This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch ml/windowSet in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1ed17c26702e94278af639a994a69927b4d7ea27 Merge: 0ffeb6c953 8755c3fbea Author: liuminghui233 <[email protected]> AuthorDate: Mon Nov 21 16:26:15 2022 +0800 merge master .github/ISSUE_TEMPLATE/bug_report.md | 31 - .github/ISSUE_TEMPLATE/bug_report.yml | 85 ++ .github/ISSUE_TEMPLATE/feature_request.md | 17 - .github/ISSUE_TEMPLATE/feature_request.yml | 66 + .github/dependabot.yml | 52 - .github/workflows/client-cpp.yml | 21 +- .github/workflows/client-go.yml | 11 +- .github/workflows/client-python.yml | 11 +- .../{cluster-it.yml => cluster-it-1c1d.yml} | 24 +- .../{cluster-it.yml => cluster-it-1c3d.yml} | 15 +- .github/workflows/e2e.yml | 6 +- .github/workflows/grafana-plugin.yml | 18 +- .github/workflows/greetings.yml | 2 +- .github/workflows/influxdb-protocol.yml | 11 +- .github/workflows/main-unix.yml | 16 +- .github/workflows/main-win.yml | 13 +- .../workflows/{client-go.yml => site-build.yml} | 34 +- .github/workflows/sonar-coveralls.yml | 36 +- .github/workflows/standalone-it-for-mpp.yml | 79 - .github/workflows/sync.yml | 9 +- .gitmodules | 4 +- LICENSE | 15 +- README.md | 19 +- README_ZH.md | 29 +- antlr/pom.xml | 17 + .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 3 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 52 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 12 + .../src/assembly/resources/conf/logback-cli.xml | 20 +- cli/src/assembly/resources/sbin/start-cli.sh | 118 +- cli/src/assembly/resources/tools/export-csv.sh | 7 + cli/src/assembly/resources/tools/export-tsfile.sh | 7 + cli/src/assembly/resources/tools/import-csv.sh | 7 + .../java/org/apache/iotdb/cli/AbstractCli.java | 21 + cli/src/main/java/org/apache/iotdb/cli/Cli.java | 5 + .../org/apache/iotdb/cli/StartClientScriptIT.java | 3 - client-go | 2 +- client-py/README.md | 12 +- client-py/SessionAlignedTimeseriesExample.py | 4 +- client-py/SessionExample.py | 772 +++++----- client-py/iotdb/Session.py | 16 +- client-py/iotdb/dbapi/tests/test_cursor.py | 8 +- client-py/iotdb/sqlalchemy/IoTDBDialect.py | 2 +- client-py/iotdb/sqlalchemy/tests/test_dialect.py | 8 +- client-py/tests/test_aligned_timeseries.py | 6 +- client-py/tests/test_dataframe.py | 24 +- client-py/tests/test_one_device.py | 2 +- client-py/tests/test_session.py | 6 +- client-py/tests/test_tablet.py | 4 +- client-py/tests/test_template.py | 2 +- client-py/tests/test_todf.py | 6 +- confignode/src/assembly/confignode.xml | 4 + .../src/assembly/resources/conf/confignode-env.bat | 27 +- .../src/assembly/resources/conf/confignode-env.sh | 10 + .../resources/conf/iotdb-confignode.properties | 41 +- .../src/assembly/resources/conf/logback-tool.xml | 16 +- .../assembly/resources/sbin/remove-confignode.sh | 59 +- .../assembly/resources/sbin/start-confignode.bat | 41 +- .../assembly/resources/sbin/start-confignode.sh | 169 ++- .../assembly/resources/sbin/stop-confignode.bat | 3 + .../src/assembly/resources/sbin/stop-confignode.sh | 4 +- .../confignode/client/ConfigNodeRequestType.java | 2 +- .../confignode/client/DataNodeRequestType.java | 1 + .../client/async/AsyncDataNodeClientPool.java | 10 + .../client/async/handlers/AsyncClientHandler.java | 1 + .../client/sync/SyncConfigNodeClientPool.java | 10 +- .../client/sync/SyncDataNodeClientPool.java | 122 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 281 ++-- .../iotdb/confignode/conf/ConfigNodeConstant.java | 6 + .../confignode/conf/ConfigNodeDescriptor.java | 622 +++++--- .../confignode/conf/ConfigNodeRemoveCheck.java | 2 +- .../confignode/conf/ConfigNodeStartupCheck.java | 22 +- .../consensus/request/ConfigPhysicalPlan.java | 7 + .../write/procedure/UpdateProcedurePlan.java | 8 - .../consensus/response/DataNodeRegisterResp.java | 20 + .../consensus/response/TemplateSetInfoResp.java | 1 + ...hine.java => ConfigNodeRegionStateMachine.java} | 139 +- ...n.java => NoAvailableRegionGroupException.java} | 15 +- .../confignode/manager/ClusterSchemaManager.java | 48 +- .../iotdb/confignode/manager/ConfigManager.java | 53 +- .../iotdb/confignode/manager/ConsensusManager.java | 107 +- .../apache/iotdb/confignode/manager/IManager.java | 4 + .../iotdb/confignode/manager/ProcedureManager.java | 14 +- .../iotdb/confignode/manager/SyncManager.java | 8 +- .../iotdb/confignode/manager/TriggerManager.java | 2 +- .../iotdb/confignode/manager/UDFManager.java | 34 +- .../iotdb/confignode/manager/load/LoadManager.java | 9 +- .../manager/load/LoadManagerMetrics.java | 10 +- .../manager/load/balancer/PartitionBalancer.java | 6 +- .../manager/load/balancer/RouteBalancer.java | 147 +- .../partition/GreedyPartitionAllocator.java | 76 +- .../balancer/partition/IPartitionAllocator.java | 6 +- .../manager/load/balancer/router/mcf/MCFEdge.java | 17 +- .../balancer/router/mcf/MCFLeaderBalancer.java | 305 ++++ .../iotdb/confignode/manager/node/NodeManager.java | 63 +- .../manager/partition/PartitionManager.java | 46 +- .../iotdb/confignode/persistence/AuthorInfo.java | 34 +- .../iotdb/confignode/persistence/TriggerInfo.java | 2 +- .../iotdb/confignode/persistence/UDFInfo.java | 2 +- .../iotdb/confignode/persistence/cq/CQInfo.java | 2 +- .../confignode/persistence/node/NodeInfo.java | 25 +- .../persistence/partition/PartitionInfo.java | 13 +- .../partition/StorageGroupPartitionTable.java | 23 +- .../persistence/schema/ClusterSchemaInfo.java | 26 +- .../persistence/schema/TemplateTable.java | 5 +- .../persistence/sync/ClusterSyncInfo.java | 14 +- .../confignode/procedure/ProcedureExecutor.java | 7 +- .../procedure/env/ConfigNodeProcedureEnv.java | 151 +- .../procedure/env/DataNodeRemoveHandler.java | 179 ++- .../impl/node/AddConfigNodeProcedure.java | 40 +- .../impl/node/RemoveDataNodeProcedure.java | 111 +- .../impl/schema/DeactivateTemplateProcedure.java | 6 + .../impl/schema/DeleteStorageGroupProcedure.java | 6 +- .../impl/schema/DeleteTimeSeriesProcedure.java | 3 +- .../impl/statemachine/RegionMigrateProcedure.java | 37 +- .../procedure/impl/sync/DropPipeProcedure.java | 10 +- .../procedure/state/AddConfigNodeState.java | 3 +- .../procedure/store/ConfigProcedureStore.java | 2 +- .../iotdb/confignode/service/ConfigNode.java | 23 +- .../thrift/ConfigNodeRPCServiceHandlerMetrics.java | 2 +- .../thrift/ConfigNodeRPCServiceMetrics.java | 2 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 23 +- .../request/ConfigPhysicalPlanSerDeTest.java | 4 +- .../balancer/router/mcf/MCFLeaderBalancerTest.java | 216 +++ .../confignode/persistence/AuthorInfoTest.java | 38 +- .../persistence/ClusterSyncInfoTest.java | 40 +- ...nfignode.properties => iotdb-common.properties} | 12 +- ...otdb-metric.yml => iotdb-confignode-metric.yml} | 4 +- .../confignode1conf/iotdb-confignode.properties | 13 +- ...nfignode.properties => iotdb-common.properties} | 11 +- ...otdb-metric.yml => iotdb-confignode-metric.yml} | 4 +- .../confignode2conf/iotdb-confignode.properties | 12 +- ...nfignode.properties => iotdb-common.properties} | 11 +- ...otdb-metric.yml => iotdb-confignode-metric.yml} | 4 +- .../confignode3conf/iotdb-confignode.properties | 12 +- consensus/README.md | 2 +- .../apache/iotdb/consensus/ConsensusFactory.java | 7 +- .../org/apache/iotdb/consensus/IStateMachine.java | 12 + .../multileader/MultiLeaderConsensus.java | 2 +- .../multileader/MultiLeaderServerImpl.java | 20 +- .../multileader/MultiLeaderServerMetrics.java | 4 +- .../multileader/client/DispatchLogHandler.java | 6 +- .../multileader/logdispatcher/LogDispatcher.java | 28 +- .../logdispatcher/MultiLeaderMemoryManager.java | 40 +- .../MultiLeaderMemoryManagerMetrics.java | 2 +- .../service/MultiLeaderRPCServiceProcessor.java | 2 +- .../ratis/ApplicationStateMachineProxy.java | 3 +- .../iotdb/consensus/ratis/RatisConsensus.java | 23 +- .../org/apache/iotdb/consensus/ratis/Utils.java | 12 +- .../SimpleConsensus.java} | 26 +- .../SimpleServerImpl.java} | 6 +- .../multileader/MultiLeaderConsensusTest.java | 4 +- .../iotdb/consensus/multileader/RecoveryTest.java | 4 +- .../iotdb/consensus/ratis/RatisConsensusTest.java | 22 +- .../apache/iotdb/consensus/ratis/UtilsTest.java | 4 +- .../{standalone => simple}/RecoveryTest.java | 6 +- .../SimpleConsensusTest.java} | 12 +- .../tests/tools/importCsv/ImportCsvTestIT.java | 6 +- .../tests/tools/tsfile/ExportTsFileTestIT.java | 6 +- .../src/test/resources/iotdb-datanode.properties | 11 +- .../src/test/resources/iotdb-engine.properties | 21 - distribution/src/assembly/all.xml | 5 + distribution/src/assembly/confignode.xml | 5 + distribution/src/assembly/datanode.xml | 5 + .../GrafanaPlugin/iotdb/conf/iotdb-rest.properties | 3 + docker/src/main/Dockerfile-1c1d | 14 +- docker/src/main/Dockerfile-1c1d-influxdb | 16 +- docs/Blog/Release0_93.md | 2 +- docs/Development/VoteRelease.md | 4 +- docs/Download/README.md | 2 +- docs/UserGuide/API/InfluxDB-Protocol.md | 12 +- docs/UserGuide/API/Interface-Comparison.md | 4 +- docs/UserGuide/API/Programming-Cpp-Native-API.md | 6 +- docs/UserGuide/API/Programming-JDBC.md | 8 +- docs/UserGuide/API/Programming-Java-Native-API.md | 14 +- .../UserGuide/API/Programming-Python-Native-API.md | 16 +- docs/UserGuide/API/Programming-Thrift.md | 2 +- docs/UserGuide/API/RestService.md | 24 +- docs/UserGuide/API/Status-Codes.md | 175 ++- .../Administration-Management/Administration.md | 18 +- docs/UserGuide/Cluster/Cluster-Concept.md | 4 +- docs/UserGuide/Cluster/Cluster-Setup.md | 50 +- docs/UserGuide/Data-Concept/Compression.md | 93 +- .../Data-Concept/Data-Model-and-Terminology.md | 20 +- .../{Compression.md => Deadband-Process.md} | 29 +- docs/UserGuide/Data-Concept/Schema-Template.md | 2 +- docs/UserGuide/Data-Concept/Time-Partition.md | 17 +- docs/UserGuide/Data-Modeling/DataRegion.md | 22 +- docs/UserGuide/Delete-Data/Delete-Data.md | 2 +- docs/UserGuide/Delete-Data/TTL.md | 26 +- .../Ecosystem-Integration/Grafana-Plugin.md | 2 +- .../Ecosystem-Integration/Zeppelin-IoTDB.md | 2 +- .../Sync-Tool.md | 44 +- .../Frequently-asked-questions.md | 0 .../Integration-Test-refactoring-tutorial.md | 12 +- .../Maintenance-Tools/Maintenance-Command.md | 103 +- docs/UserGuide/Maintenance-Tools/NodeTool.md | 14 +- .../UserGuide/{Alert => Monitor-Alert}/Alerting.md | 2 +- .../Metric-Tool.md | 190 ++- .../Operate-Metadata/Auto-Create-MetaData.md | 10 +- docs/UserGuide/Operate-Metadata/Node.md | 20 +- docs/UserGuide/Operate-Metadata/Storage-Group.md | 73 +- docs/UserGuide/Operate-Metadata/Template.md | 36 +- docs/UserGuide/Operate-Metadata/Timeseries.md | 36 +- docs/UserGuide/Process-Data/Continuous-Query.md | 4 +- docs/UserGuide/Process-Data/Select-Into.md | 6 +- docs/UserGuide/{Alert => Process-Data}/Triggers.md | 33 +- .../Process-Data/UDF-User-Defined-Function.md | 20 +- docs/UserGuide/Query-Data/Aggregate-Query.md | 6 +- docs/UserGuide/Query-Data/Select-Expression.md | 9 +- .../UserGuide/QuickStart/Command-Line-Interface.md | 8 +- docs/UserGuide/QuickStart/QuickStart.md | 49 +- docs/UserGuide/QuickStart/ServerFileList.md | 53 +- .../Reference/ConfigNode-Config-Manual.md | 36 +- docs/UserGuide/Reference/DataNode-Config-Manual.md | 26 +- docs/UserGuide/Reference/Keywords.md | 2 + docs/UserGuide/Reference/SQL-Reference.md | 50 +- docs/UserGuide/Reference/Syntax-Conventions.md | 42 +- docs/UserGuide/Reference/TSDB-Comparison.md | 4 +- docs/UserGuide/UDF-Library/Data-Profiling.md | 73 + docs/UserGuide/UDF-Library/Data-Quality.md | 57 +- docs/UserGuide/UDF-Library/M4.md | 2 +- docs/UserGuide/Write-Data/CSV-Tool.md | 6 +- docs/UserGuide/Write-Data/Load-External-Tsfile.md | 2 +- .../UserGuide/Write-Data/MQTT.md | 10 +- .../{UDF-Library/M4.md => Write-Data/REST-API.md} | 37 +- .../{UDF-Library/M4.md => Write-Data/Session.md} | 16 +- docs/zh/Development/VoteRelease.md | 4 +- docs/zh/Download/README.md | 2 +- docs/zh/UserGuide/API/InfluxDB-Protocol.md | 12 +- docs/zh/UserGuide/API/Interface-Comparison.md | 4 +- .../zh/UserGuide/API/Programming-Cpp-Native-API.md | 6 +- docs/zh/UserGuide/API/Programming-JDBC.md | 8 +- .../UserGuide/API/Programming-Java-Native-API.md | 12 +- .../UserGuide/API/Programming-Python-Native-API.md | 14 +- docs/zh/UserGuide/API/Programming-Thrift.md | 4 +- docs/zh/UserGuide/API/RestService.md | 22 +- docs/zh/UserGuide/API/Status-Codes.md | 176 ++- .../Administration-Management/Administration.md | 18 +- docs/zh/UserGuide/Cluster/Cluster-Concept.md | 4 +- docs/zh/UserGuide/Cluster/Cluster-Setup.md | 50 +- docs/zh/UserGuide/Data-Concept/Compression.md | 88 +- .../Data-Concept/Data-Model-and-Terminology.md | 22 +- .../{Compression.md => Deadband-Process.md} | 26 +- docs/zh/UserGuide/Data-Concept/Schema-Template.md | 4 +- docs/zh/UserGuide/Data-Concept/Time-Partition.md | 25 +- docs/zh/UserGuide/Data-Modeling/DataRegion.md | 12 +- docs/zh/UserGuide/Delete-Data/Delete-Data.md | 2 +- docs/zh/UserGuide/Delete-Data/TTL.md | 22 +- .../Ecosystem-Integration/Grafana-Plugin.md | 2 +- .../Ecosystem-Integration/Zeppelin-IoTDB.md | 2 +- .../Sync-Tool.md | 39 +- .../Frequently-asked-questions.md | 0 .../Integration-Test-refactoring-tutorial.md | 8 +- .../Maintenance-Tools/Maintenance-Command.md | 84 +- docs/zh/UserGuide/Maintenance-Tools/NodeTool.md | 10 +- .../Maintenance-Tools/SchemaFileSketch-Tool.md | 2 +- .../UserGuide/{Alert => Monitor-Alert}/Alerting.md | 2 +- .../Metric-Tool.md | 222 +-- .../Operate-Metadata/Auto-Create-MetaData.md | 10 +- docs/zh/UserGuide/Operate-Metadata/Node.md | 18 +- .../zh/UserGuide/Operate-Metadata/Storage-Group.md | 73 +- docs/zh/UserGuide/Operate-Metadata/Template.md | 22 +- docs/zh/UserGuide/Operate-Metadata/Timeseries.md | 18 +- docs/zh/UserGuide/Process-Data/Continuous-Query.md | 4 +- docs/zh/UserGuide/Process-Data/Select-Into.md | 6 +- .../UserGuide/{Alert => Process-Data}/Triggers.md | 28 +- .../Process-Data/UDF-User-Defined-Function.md | 18 +- docs/zh/UserGuide/Query-Data/Aggregate-Query.md | 8 +- docs/zh/UserGuide/Query-Data/Select-Expression.md | 9 +- .../UserGuide/QuickStart/Command-Line-Interface.md | 8 +- docs/zh/UserGuide/QuickStart/QuickStart.md | 36 +- docs/zh/UserGuide/QuickStart/ServerFileList.md | 46 +- .../Reference/ConfigNode-Config-Manual.md | 28 +- .../UserGuide/Reference/DataNode-Config-Manual.md | 22 +- docs/zh/UserGuide/Reference/Keywords.md | 2 + docs/zh/UserGuide/Reference/SQL-Reference.md | 60 +- docs/zh/UserGuide/Reference/Syntax-Conventions.md | 44 +- docs/zh/UserGuide/UDF-Library/Data-Profiling.md | 74 + docs/zh/UserGuide/UDF-Library/Data-Quality.md | 57 +- docs/zh/UserGuide/Write-Data/CSV-Tool.md | 6 +- .../UserGuide/Write-Data/Load-External-Tsfile.md | 2 +- .../zh/UserGuide/Write-Data/MQTT.md | 10 +- .../M4.md => zh/UserGuide/Write-Data/REST-API.md} | 38 +- .../M4.md => zh/UserGuide/Write-Data/Session.md} | 16 +- .../src/AlignedTimeseriesSessionExample.cpp | 2 +- .../org/apache/iotdb/flink/FlinkIoTDBSource.java | 2 +- .../main/java/org/apache/iotdb/JDBCExample.java | 2 +- .../iotdb/SyntaxConventionRelatedExample.java | 4 +- example/mqtt/README.md | 2 +- example/rest-client-c-example/main.c | 4 +- example/rocketmq/readme.md | 6 +- .../org/apache/iotdb/DataMigrationExample.java | 4 +- .../main/java/org/apache/iotdb/SessionExample.java | 89 +- .../iotdb/SyntaxConventionRelatedExample.java | 8 +- example/trigger/pom.xml | 2 +- .../java/org/apache/iotdb/tsfile/TsFileRead.java | 11 +- example/udf/pom.xml | 3 +- .../SlidingTimeWindowAccessStrategyExample.java | 105 ++ .../java/org/apache/iotdb/udf/UDTFExample.java | 2 +- .../Apache IoTDB ConfigNode Dashboard v0.14.0.json | 238 +-- .../Apache IoTDB DataNode Dashboard v0.14.0.json | 121 +- .../standalone/Apache IoTDB Dashboard v0.14.0.json | 2 +- grafana-plugin/README.md | 130 +- grafana-plugin/pkg/main.go | 2 +- .../pkg/plugin/iotdb_resource_handler.go | 170 +++ grafana-plugin/pkg/plugin/plugin.go | 119 +- grafana-plugin/src/ConfigEditor.tsx | 70 +- grafana-plugin/src/datasource.ts | 47 +- grafana-plugin/src/img/addIoTDBDataSource.png | Bin 0 -> 43760 bytes grafana-plugin/src/img/showData.png | Bin 0 -> 170827 bytes grafana-plugin/src/plugin.json | 4 +- grafana-plugin/src/types.ts | 3 +- .../iotdb/hadoop/tsfile/TSFRecordReader.java | 2 +- integration-test/README.md | 61 +- integration-test/import-control.xml | 6 +- integration-test/pom.xml | 23 +- .../java/org/apache/iotdb/it/env/AbstractEnv.java | 15 + .../apache/iotdb/it/env/AbstractNodeWrapper.java | 8 +- .../org/apache/iotdb/it/env/ConfigFactory.java | 41 +- .../org/apache/iotdb/it/env/ConfigNodeWrapper.java | 8 +- .../org/apache/iotdb/it/env/DataNodeWrapper.java | 3 +- .../java/org/apache/iotdb/it/env/EnvFactory.java | 20 +- .../main/java/org/apache/iotdb/it/env/EnvType.java | 14 +- .../java/org/apache/iotdb/it/env/MppConfig.java | 39 +- .../org/apache/iotdb/it/env/RemoteServerEnv.java | 8 +- .../{StandaloneOnMppEnv.java => SimpleEnv.java} | 22 +- .../apache/iotdb/itbase/constant/TestConstant.java | 11 +- .../org/apache/iotdb/itbase/env/BaseConfig.java | 40 +- .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 53 + .../confignode/it/IoTDBClusterAuthorityIT.java | 40 +- .../iotdb/confignode/it/IoTDBClusterNodeIT.java | 2 +- .../it/IoTDBClusterRegionLeaderBalancingIT.java | 155 +- .../iotdb/confignode/it/IoTDBClusterRestartIT.java | 6 +- .../confignode/it/IoTDBConfigNodeSnapshotIT.java | 8 +- .../it/IoTDBConfigNodeSwitchLeaderIT.java | 14 +- .../iotdb/confignode/it/IoTDBStorageGroupIT.java | 3 +- .../it/partition/IoTDBPartitionDurableTest.java | 323 ++++ .../IoTDBPartitionGetterIT.java} | 592 ++------ .../partition/IoTDBPartitionInheritPolicyTest.java | 182 +++ .../confignode/it/utils/ConfigNodeTestUtils.java | 66 + .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 33 +- .../java/org/apache/iotdb/db/it/IoTDBAuthIT.java | 45 +- .../apache/iotdb/db/it}/IoTDBDatetimeFormatIT.java | 33 +- .../org/apache/iotdb/db/it/IoTDBDeletionIT.java | 38 +- .../org/apache/iotdb/db/it}/IoTDBEncodingIT.java | 220 +-- .../org/apache/iotdb/db/it/IoTDBExampleIT.java | 6 +- .../apache/iotdb/db/it}/IoTDBExecuteBatchIT.java | 77 +- .../apache/iotdb/db/it/IoTDBFilterBetweenIT.java | 20 +- .../java/org/apache/iotdb/db/it/IoTDBFilterIT.java | 64 +- .../org/apache/iotdb/db/it/IoTDBFilterNullIT.java | 15 +- .../apache/iotdb/db/it}/IoTDBFloatPrecisionIT.java | 47 +- .../apache/iotdb/db/it/IoTDBFlushQueryMergeIT.java | 28 +- .../apache/iotdb/db/it}/IoTDBInsertMultiRowIT.java | 52 +- .../org/apache/iotdb/db/it}/IoTDBInsertNaNIT.java | 56 +- .../org/apache/iotdb/db/it}/IoTDBInsertNullIT.java | 44 +- .../iotdb/db/it}/IoTDBInsertWithQueryIT.java | 31 +- .../iotdb/db/it/IoTDBInsertWithoutTimeIT.java | 2 +- .../org/apache/iotdb/db/it/IoTDBMultiDeviceIT.java | 98 +- .../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 15 +- .../iotdb/db/it}/IoTDBPartialInsertionIT.java | 50 +- .../it}/IoTDBQueryWithRecreatedTimeseriesIT.java | 23 +- .../org/apache/iotdb/db/it/IoTDBQuotedPathIT.java | 7 +- .../org/apache/iotdb/db/it}/IoTDBRecoverIT.java | 147 +- .../iotdb/db/it}/IoTDBRecoverUnclosedIT.java | 131 +- .../org/apache/iotdb/db/it}/IoTDBRestartIT.java | 221 ++- .../it/IoTDBSameMeasurementsDifferentTypesIT.java | 2 +- .../org/apache/iotdb/db/it}/IoTDBSettleIT.java | 27 +- .../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 80 +- .../db/it/IoTDBSyntaxConventionIdentifierIT.java | 275 ++-- .../it/IoTDBSyntaxConventionStringLiteralIT.java | 53 +- .../org/apache/iotdb/db/it/IoTDBTimeZoneIT.java | 6 +- .../java/org/apache/iotdb/db/it/IoTDBTtlIT.java | 19 +- .../it/aggregation/IoTDBAggregationByLevelIT.java | 35 +- .../db/it/aggregation/IoTDBAggregationIT.java | 70 +- .../aggregation/IoTDBAggregationLargeDataIT.java | 2 +- .../aggregation/IoTDBAggregationSmallDataIT.java | 23 +- .../db/it/aggregation/IoTDBTagAggregationIT.java | 6 +- .../db/it/alignbydevice/IoTDBAlignByDeviceIT.java | 4 +- .../aligned/IoTDBAggregationWithDeletion2IT.java | 4 +- .../it/aligned/IoTDBAggregationWithDeletionIT.java | 7 +- .../db/it/aligned/IoTDBAlignedDataDeletionIT.java | 36 +- .../db/it/aligned/IoTDBAlignedLastQuery2IT.java | 4 +- .../db/it/aligned/IoTDBAlignedLastQuery3IT.java | 4 +- .../db/it/aligned/IoTDBAlignedLastQueryIT.java | 13 +- .../db/it/aligned/IoTDBAlignedSeriesQueryIT.java | 104 +- .../db/it/aligned/IoTDBGroupByLevelQueryIT.java | 2 +- ...GroupByQueryWithValueFilterWithDeletion2IT.java | 4 +- ...BGroupByQueryWithValueFilterWithDeletionIT.java | 4 +- ...upByQueryWithoutValueFilterWithDeletion2IT.java | 4 +- ...oupByQueryWithoutValueFilterWithDeletionIT.java | 4 +- .../db/it/aligned/IoTDBInsertAlignedValues2IT.java | 10 +- .../db/it/aligned/IoTDBInsertAlignedValues3IT.java | 10 +- .../db/it/aligned/IoTDBInsertAlignedValues4IT.java | 4 +- .../db/it/aligned/IoTDBInsertAlignedValuesIT.java | 41 +- .../it/aligned/IoTDBLastQueryWithDeletion2IT.java | 8 +- .../it/aligned/IoTDBLastQueryWithDeletionIT.java | 8 +- .../aligned/IoTDBLastQueryWithoutLastCache2IT.java | 4 +- .../aligned/IoTDBLastQueryWithoutLastCacheIT.java | 4 +- ...DBLastQueryWithoutLastCacheWithDeletion2IT.java | 8 +- ...TDBLastQueryWithoutLastCacheWithDeletionIT.java | 8 +- ...BRawQueryWithoutValueFilterWithDeletion2IT.java | 3 +- ...DBRawQueryWithoutValueFilterWithDeletionIT.java | 18 +- .../org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java | 3 +- .../java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java | 53 +- .../iotdb/db/it/env/StandaloneEnvConfig.java | 23 +- .../apache/iotdb/db/it/groupby/IOTDBGroupByIT.java | 2 +- .../iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java | 4 +- .../apache/iotdb/db/it/groupby/IoTDBHavingIT.java | 18 +- .../IoTDBLastWithTTLIT.java} | 63 +- .../org/apache/iotdb/db/it/query/IoTDBAliasIT.java | 6 +- .../iotdb/db/it/query}/IoTDBArithmeticIT.java | 151 +- .../iotdb/db/it/query}/IoTDBFuzzyQueryIT.java | 161 +- .../org/apache/iotdb/db/it/query}/IoTDBInIT.java | 76 +- .../iotdb/db/it/query/IoTDBNullOperandIT.java | 243 +++ .../iotdb/db/it/query/IoTDBNullValueFillIT.java | 5 +- .../iotdb/db/it/query/IoTDBPaginationIT.java | 2 +- .../iotdb/db/it/query}/IoTDBQueryDemoIT.java | 631 ++++---- .../query/IoTDBQueryWithComplexValueFilterIT.java | 2 +- .../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 36 +- .../it/query}/IoTDBSelectCompareExpressionIT.java | 15 +- .../iotdb/db/it/query}/IoTDBSelectSchemaIT.java | 15 +- .../db/it/schema/IoTDBAutoCreateSchemaIT.java | 40 +- .../it/schema/IoTDBCreateAlignedTimeseriesIT.java | 21 +- .../db/it/schema/IoTDBCreateStorageGroupIT.java | 31 +- .../db/it/schema/IoTDBCreateTimeseriesIT.java | 12 +- .../db/it/schema/IoTDBDeactivateTemplateIT.java | 35 +- .../it/schema/IoTDBDeleteAlignedTimeseriesIT.java | 7 +- .../db/it/schema/IoTDBDeleteStorageGroupIT.java | 54 +- .../db/it/schema/IoTDBDeleteTimeseriesIT.java | 13 +- .../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 99 +- .../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 97 +- .../db/it/schema/IoTDBSortedShowTimeseriesIT.java | 68 +- .../apache/iotdb/db/it/schema/IoTDBTagAlterIT.java | 208 +-- .../org/apache/iotdb/db/it/schema/IoTDBTagIT.java | 275 ++-- .../iotdb/db/it/selectinto/IoTDBSelectInto2IT.java | 4 +- .../iotdb/db/it/selectinto/IoTDBSelectInto3IT.java | 4 +- .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 24 +- .../org/apache/iotdb/db/it/sync/IoTDBPipeIT.java | 21 +- .../apache/iotdb/db/it/sync/IoTDBPipeSinkIT.java | 23 +- .../db/it/trigger/IoTDBTriggerExecutionIT.java | 6 +- .../db/it/trigger/IoTDBTriggerManagementIT.java | 169 ++- .../org/apache/iotdb/db/it/udf/IoTDBUDFM4IT.java | 2 +- .../iotdb/db/it/udf/IoTDBUDFManagementIT.java | 28 + .../db/it/udf/IoTDBUDFOtherWindowQueryIT.java | 86 +- .../iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java | 33 +- .../db/it/udf/IoTDBUDTFAlignByTimeQueryIT.java | 4 +- .../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 15 +- .../iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java | 6 +- .../iotdb/db/it/udf/IoTDBUDTFNonAlignQueryIT.java | 2 +- .../apache/iotdb/db/it/utils/AlignedWriteUtil.java | 6 +- .../org/apache/iotdb/db/it/utils/TestUtils.java | 4 +- .../iotdb/db/it/watermark/IoTDBWatermarkIT.java | 2 +- .../db/it/withoutNull/IoTDBWithoutAllNullIT.java | 17 +- .../db/it/withoutNull/IoTDBWithoutAnyNullIT.java | 5 +- .../withoutNull/IoTDBWithoutNullAllFilterIT.java | 44 +- .../withoutNull/IoTDBWithoutNullAnyFilterIT.java | 60 +- .../session/it/IoTDBSessionAlignedInsertIT.java | 95 +- .../iotdb/session/it/IoTDBSessionComplexIT.java | 76 +- .../session/it/IoTDBSessionSchemaTemplateIT.java | 12 +- .../iotdb/session/it/IoTDBSessionSimpleIT.java | 26 +- .../iotdb/session/it/pool/SessionPoolIT.java | 46 +- .../src/test/resources/iotdb-datanode.properties | 10 +- .../src/test/resources/iotdb-engine.properties | 21 - .../iotdb/integration/env/ClusterEnvBase.java | 4 +- .../iotdb/integration/env/ClusterEnvConfig.java | 7 +- .../iotdb/integration/env/RemoteServerEnv.java | 8 +- .../org/apache/iotdb/itbase/env/BaseConfig.java | 4 - .../apache/iotdb/db/integration/IOTDBInsertIT.java | 2 +- .../IoTDBAlignedTimeSeriesCompactionIT.java | 4 +- .../iotdb/db/integration/IoTDBAuthorizationIT.java | 54 +- .../iotdb/db/integration/IoTDBClearCacheIT.java | 4 +- .../apache/iotdb/db/integration/IoTDBCloseIT.java | 2 +- .../iotdb/db/integration/IoTDBCompactionIT.java | 11 +- .../integration/IoTDBCompactionWithIDTableIT.java | 11 +- .../apache/iotdb/db/integration/IoTDBDaemonIT.java | 4 +- .../iotdb/db/integration/IoTDBDisableAlignIT.java | 4 +- .../db/integration/IoTDBEngineTimeGeneratorIT.java | 2 +- .../iotdb/db/integration/IoTDBFilePathUtilsIT.java | 99 -- .../apache/iotdb/db/integration/IoTDBFillIT.java | 4 +- .../db/integration/IoTDBFlushQueryMergeIT.java | 18 +- .../db/integration/IoTDBLoadExternalTsfileIT.java | 15 +- .../IoTDBLoadExternalTsfileWithVirtualSGIT.java | 7 +- .../integration/IoTDBManageTsFileResourceIT.java | 291 ---- .../iotdb/db/integration/IoTDBMultiSeriesIT.java | 5 +- .../db/integration/IoTDBMultiStatementsIT.java | 2 +- .../db/integration/IoTDBNewTsFileCompactionIT.java | 2 +- .../db/integration/IoTDBOverlappedPageIT.java | 2 +- .../db/integration/IoTDBQueryMemoryControlIT.java | 2 +- .../iotdb/db/integration/IoTDBQueryTimeoutIT.java | 4 +- .../db/integration/IoTDBQueryWithIDTableIT.java | 4 +- .../db/integration/IoTDBRemovePartitionIT.java | 620 ++++---- .../db/integration/IoTDBRewriteTsFileToolIT.java | 2 +- .../db/integration/IoTDBRpcCompressionIT.java | 2 +- .../db/integration/IoTDBSchemaTemplateIT.java | 87 +- .../iotdb/db/integration/IoTDBSensorUpdateIT.java | 2 +- .../db/integration/IoTDBSequenceDataQueryIT.java | 2 +- .../iotdb/db/integration/IoTDBSeriesReaderIT.java | 9 +- .../db/integration/IoTDBSessionTimeoutIT.java | 2 +- .../IoTDBSetSystemReadOnlyWritableIT.java | 4 +- .../integration/IoTDBSizeTieredCompactionIT.java | 44 +- .../iotdb/db/integration/IoTDBSnapshotIT.java | 8 +- .../integration/IoTDBSortedShowTimeseriesIT.java | 69 +- .../iotdb/db/integration/IoTDBTimePartitionIT.java | 5 +- .../iotdb/db/integration/IoTDBTracingIT.java | 2 +- .../db/integration/IoTDBTriggerExecutionIT.java | 678 --------- .../db/integration/IoTDBTriggerForwardIT.java | 305 ---- .../db/integration/IoTDBTriggerManagementIT.java | 539 ------- .../apache/iotdb/db/integration/IoTDBTtlIT.java | 40 +- .../db/integration/IoTDBUDFWindowQueryIT.java | 2 +- .../db/integration/IoTDBUDTFHybridQueryIT.java | 2 +- .../db/integration/IoTDBUDTFNonAlignQueryIT.java | 2 +- .../db/integration/IoTDBUnseqOverlappedPageIT.java | 2 +- .../iotdb/db/integration/IoTDBVersionIT.java | 4 +- .../aggregation/IoTDBUDFNestAggregationIT.java | 17 +- .../IoTDBUserDefinedAggregationFunctionIT.java | 16 +- .../db/integration/aligned/AlignedWriteUtil.java | 2 +- .../aligned/IoTDBGroupByFillQueryBigDataIT.java | 5 +- .../aligned/IoTDBGroupByFillQueryIT.java | 3 +- .../aligned/IoTDBLoadExternalAlignedTsFileIT.java | 15 +- .../db/integration/env/StandaloneEnvConfig.java | 7 +- .../db/integration/groupby/IoTDBGroupByFillIT.java | 4 +- .../groupby/IoTDBGroupByFillMixPathsIT.java | 4 +- .../groupby/IoTDBGroupByFillWithRangeIT.java | 2 +- .../db/integration/sync/IoTDBSyncReceiverIT.java | 59 - .../sync/IoTDBSyncReceiverLoaderIT.java | 76 +- .../db/integration/sync/IoTDBSyncSenderIT.java | 58 +- .../iotdb/db/integration/sync/SyncTestUtil.java | 4 +- .../versionadaption/IoTDBDDLVersionAdaptionIT.java | 54 +- .../IoTDBDeletionVersionAdaptionIT.java | 7 +- .../IoTDBQueryVersionAdaptionIT.java | 4 +- .../session/IoTDBSessionAlignedABDeviceIT.java | 6 +- .../session/IoTDBSessionAlignedAggregationIT.java | 6 +- .../IoTDBSessionAlignedAggregationWithUnSeqIT.java | 6 +- .../src/test/resources/iotdb-datanode.properties | 10 +- .../src/test/resources/iotdb-engine.properties | 21 - jdbc/README.md | 10 +- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 86 +- .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 12 +- .../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 4 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 9 +- .../test/java/org/apache/iotdb/jdbc/BatchTest.java | 2 +- .../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 6 +- .../apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java | 129 +- library-udf/pom.xml | 115 +- library-udf/src/assembly/tools/register-UDF.bat | 1 + library-udf/src/assembly/tools/register-UDF.sh | 1 + .../iotdb/library/dprofile/UDAFQuantile.java | 127 ++ .../library/dprofile/util/HeapLongKLLSketch.java | 356 +++++ .../dprofile/util/KLLSketchForQuantile.java | 172 +++ metrics/dropwizard-metrics/pom.xml | 2 +- .../resources/conf/iotdb-confignode-metric.yml | 8 +- .../resources/conf/iotdb-datanode-metric.yml | 8 +- .../apache/iotdb/metrics/config/MetricConfig.java | 25 +- .../iotdb/metrics/utils/IoTDBMetricsUtils.java | 7 +- .../iotdb/metrics/config/MetricConfigTest.java | 2 +- .../interface/src/test/resources/iotdb-metric.yml | 2 +- metrics/micrometer-metrics/pom.xml | 2 +- .../micrometer/MicrometerMetricManager.java | 2 + .../resources/conf/iotdb-common.properties | 1258 +++++++--------- .../src/assembly/resources/sbin/iotdb-common.sh | 211 +++ .../commons/auth/authorizer/OpenIdAuthorizer.java | 41 +- .../org/apache/iotdb/commons/auth/entity/User.java | 13 +- .../iotdb/commons/concurrent/ThreadName.java | 2 +- .../apache/iotdb/commons/conf/CommonConfig.java | 9 +- .../iotdb/commons/conf/CommonDescriptor.java | 61 +- .../apache/iotdb/commons/conf/IoTDBConstant.java | 55 +- ...titionRegionId.java => ConfigNodeRegionId.java} | 6 +- .../iotdb/commons/consensus/ConsensusGroupId.java | 8 +- .../commons/exception/ConfigurationException.java | 6 +- .../commons/exception/IllegalPathException.java | 4 +- .../commons/exception/sync/PipeSinkException.java | 4 +- .../iotdb/commons/partition/DataPartition.java | 10 +- .../iotdb/commons/partition/SchemaPartition.java | 3 +- .../org/apache/iotdb/commons/path/PartialPath.java | 6 +- .../apache/iotdb/commons/path/PathPatternNode.java | 58 +- .../apache/iotdb/commons/path/PathPatternTree.java | 16 +- .../iotdb/commons/sync/metadata/SyncMetadata.java | 2 +- .../iotdb/commons/sync/pipe/TsFilePipeInfo.java | 5 +- .../iotdb/commons/sync/pipesink/IoTDBPipeSink.java | 29 +- .../iotdb/commons/sync/pipesink/PipeSink.java | 5 - .../iotdb/commons/sync/utils/SyncConstant.java | 7 +- .../org/apache/iotdb/commons/udf/UDFTable.java | 2 +- .../commons/udf/service/UDFManagementService.java | 27 +- .../org/apache/iotdb/commons/utils/AuthUtils.java | 20 +- .../apache/iotdb/commons/utils/JVMCommonUtils.java | 2 +- .../apache/iotdb/commons/utils/ProcessIdUtils.java | 26 +- .../apache/iotdb/commons/utils/StatusUtils.java | 155 +- .../iotdb/commons/path/PathPatternTreeTest.java | 19 + .../commons/sync/metedata/SyncMetadataTest.java | 4 +- .../iotdb/commons/utils/JVMCommonUtilsTest.java | 45 +- .../commons/utils/ThriftCommonsSerDeUtilsTest.java | 2 +- pom.xml | 22 +- .../schemaregion/rocksdb/RSchemaRegion.java | 204 +-- .../schemaregion/rocksdb/mnode/RInternalMNode.java | 24 - .../schemaregion/rocksdb/mnode/RMNode.java | 20 - .../rocksdb/mnode/RMeasurementMNode.java | 26 - schema-engine-tag/README.md | 8 +- .../metadata/tagSchemaRegion/TagSchemaRegion.java | 127 +- .../utils/PathTagConverterUtils.java | 6 +- .../utils/ShowTimeSeriesResultUtils.java | 12 +- .../src/assembly/resources/conf/datanode-env.bat | 25 +- server/src/assembly/resources/conf/datanode-env.sh | 10 + .../resources/conf/iotdb-datanode.properties | 49 +- .../assembly/resources/conf/iotdb-rest.properties | 3 + .../src/assembly/resources/conf/logback-tool.xml | 16 +- .../src/assembly/resources/sbin/remove-datanode.sh | 45 +- .../src/assembly/resources/sbin/start-datanode.bat | 42 +- .../src/assembly/resources/sbin/start-datanode.sh | 194 ++- .../assembly/resources/sbin/start-new-server.bat | 55 +- .../assembly/resources/sbin/start-new-server.sh | 182 ++- .../src/assembly/resources/sbin/start-server.bat | 126 -- server/src/assembly/resources/sbin/start-server.sh | 151 -- .../src/assembly/resources/sbin/stop-datanode.bat | 3 + .../src/assembly/resources/sbin/stop-datanode.sh | 6 +- server/src/assembly/resources/sbin/stop-server.bat | 38 - server/src/assembly/resources/sbin/stop-server.sh | 74 - .../assembly/resources/tools/detect-watermark.sh | 9 +- .../assembly/resources/tools/rocksdb-transfer.bat | 2 +- .../assembly/resources/tools/rocksdb-transfer.sh | 2 +- .../resources/tools/schema/SchemaFileSketcher.sh | 9 +- .../assembly/resources/tools/schema/mLogParser.sh | 9 +- .../assembly/resources/tools/start-WalChecker.bat | 2 +- .../assembly/resources/tools/start-WalChecker.sh | 11 +- .../tools/tsfileToolSet/print-iotdb-data-dir.sh | 10 +- .../tsfileToolSet/print-tsfile-resource-files.sh | 11 +- .../tools/tsfileToolSet/print-tsfile-sketch.bat | 7 +- .../tools/tsfileToolSet/print-tsfile-sketch.sh | 10 +- .../tools/tsfileToolSet/print-tsfile-visdata.sh | 9 +- .../resources/tools/tsfileToolSet/settle.sh | 9 +- .../tools/tsfileToolSet/split-tsfile-tool.sh | 9 +- .../tools/tsfileToolSet/validate-tsfile.sh | 9 +- server/src/assembly/server.xml | 17 + .../org/apache/iotdb/db/auth/AuthorityChecker.java | 19 +- .../iotdb/db/auth/ClusterAuthorityFetcher.java | 70 +- .../iotdb/db/auth/StandaloneAuthorityFetcher.java | 5 +- .../apache/iotdb/db/client/ConfigNodeClient.java | 35 +- .../org/apache/iotdb/db/client/ConfigNodeInfo.java | 4 +- .../iotdb/db/client/DataNodeClientPoolFactory.java | 14 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 283 ++-- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 337 ++--- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 63 +- .../db/conf/directories/DirectoryChecker.java | 106 ++ .../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 11 + .../db/conf/rest/IoTDBRestServiceDescriptor.java | 3 + .../db/consensus/DataRegionConsensusImpl.java | 2 +- .../statemachine/DataRegionStateMachine.java | 16 + .../statemachine/SchemaRegionStateMachine.java | 2 +- .../statemachine/visitor/DataExecutionVisitor.java | 42 +- .../org/apache/iotdb/db/constant/TestConstant.java | 2 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 1017 +------------ .../apache/iotdb/db/engine/StorageEngineV2.java | 231 +-- .../db/engine/cache/TimeSeriesMetadataCache.java | 5 +- .../engine/compaction/CompactionTaskManager.java | 20 +- .../RewriteCrossSpaceCompactionSelector.java | 8 +- .../compaction/inner/InnerSpaceCompactionTask.java | 7 +- .../sizetiered/SizeTieredCompactionSelector.java | 78 +- .../inner/utils/MultiTsFileDeviceIterator.java | 4 +- .../db/engine/compaction/log/TsFileIdentifier.java | 10 +- .../impl/ReadChunkCompactionPerformer.java | 2 +- .../writer/CrossSpaceCompactionWriter.java | 2 +- .../writer/InnerSpaceCompactionWriter.java | 2 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 20 +- .../db/engine/flush/pool/FlushTaskPoolManager.java | 4 +- .../iotdb/db/engine/load/AlignedChunkData.java | 339 ++--- .../org/apache/iotdb/db/engine/load/ChunkData.java | 25 +- .../apache/iotdb/db/engine/load/DeletionData.java | 4 +- .../iotdb/db/engine/load/LoadTsFileManager.java | 17 +- .../iotdb/db/engine/load/NonAlignedChunkData.java | 258 ++-- .../apache/iotdb/db/engine/load/TsFileData.java | 3 +- .../load/TsFileSplitter.java} | 431 ++---- .../iotdb/db/engine/memtable/AbstractMemTable.java | 204 +-- .../engine/memtable/AlignedWritableMemChunk.java | 10 + .../memtable/AlignedWritableMemChunkGroup.java | 5 + .../apache/iotdb/db/engine/memtable/IMemTable.java | 25 +- .../db/engine/memtable/IWritableMemChunk.java | 6 +- .../db/engine/memtable/IWritableMemChunkGroup.java | 2 + .../iotdb/db/engine/memtable/WritableMemChunk.java | 9 +- .../db/engine/memtable/WritableMemChunkGroup.java | 25 +- .../db/engine/querycontext/QueryDataSource.java | 9 +- .../selectinto/InsertTabletPlanGenerator.java | 252 ---- .../selectinto/InsertTabletPlansIterator.java | 190 --- .../iotdb/db/engine/snapshot/SnapshotLoader.java | 2 +- .../iotdb/db/engine/snapshot/SnapshotLogger.java | 2 +- .../iotdb/db/engine/snapshot/SnapshotTaker.java | 18 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 871 +++-------- .../db/engine/storagegroup/DataRegionInfo.java | 6 +- ...hTimeManager.java => HashLastFlushTimeMap.java} | 236 ++- ...meManager.java => IDTableLastFlushTimeMap.java} | 180 +-- ...lushTimeManager.java => ILastFlushTimeMap.java} | 34 +- .../db/engine/storagegroup/TimePartitionInfo.java | 66 + .../engine/storagegroup/TimePartitionManager.java | 165 +++ .../db/engine/storagegroup/TsFileManager.java | 6 + .../db/engine/storagegroup/TsFileProcessor.java | 220 +-- .../db/engine/storagegroup/TsFileResource.java | 22 +- .../engine/storagegroup/TsFileResourceStatus.java | 3 +- .../dataregion/HashVirtualPartitioner.java | 58 - .../dataregion/StorageGroupManager.java | 505 ------- .../storagegroup/timeindex/DeviceTimeIndex.java | 15 +- .../storagegroup/timeindex/FileTimeIndex.java | 11 +- .../engine/storagegroup/timeindex/ITimeIndex.java | 5 + .../storagegroup/timeindex/V012FileTimeIndex.java | 6 + .../db/engine/trigger/builtin/ForwardTrigger.java | 2 +- .../db/engine/trigger/executor/TriggerEngine.java | 128 -- .../engine/trigger/executor/TriggerExecutor.java | 250 ---- .../trigger/service/TriggerClassLoaderManager.java | 95 -- .../engine/trigger/service/TriggerLogReader.java | 48 - .../engine/trigger/service/TriggerLogWriter.java | 68 - .../service/TriggerRegistrationInformation.java | 82 - .../service/TriggerRegistrationService.java | 505 ------- .../trigger/sink/local/LocalIoTDBHandler.java | 12 +- .../version/SimpleFileVersionController.java | 2 +- .../iotdb/db/engine/version/VersionController.java | 12 +- .../db/exception/ContinuousQueryException.java | 36 - .../iotdb/db/exception/DataRegionException.java | 4 +- .../exception/DiskSpaceInsufficientException.java | 2 +- .../apache/iotdb/db/exception/MergeException.java | 2 +- .../exception/StorageEngineReadonlyException.java | 2 +- .../db/exception/SyncConnectionException.java | 6 +- .../iotdb/db/exception/SystemCheckException.java | 34 - .../index/UnsupportedIndexTypeException.java | 32 - .../metadata/AliasAlreadyExistException.java | 2 +- .../exception/metadata/DeleteFailedException.java | 40 - .../metadata/MeasurementAlreadyExistException.java | 2 +- .../metadata/MeasurementInBlackListException.java | 2 +- .../metadata/PathAlreadyExistException.java | 2 +- .../exception/metadata/PathNotExistException.java | 24 +- .../SchemaDirCreationFailureException.java | 2 +- .../metadata/StorageGroupAlreadySetException.java | 8 +- .../metadata/StorageGroupNotSetException.java | 10 +- .../schemafile/ColossalRecordException.java | 6 +- .../SchemaFileLogCorruptedException.java | 2 +- .../template/NoTemplateOnMNodeException.java | 2 +- .../template/TemplateImcompatibeException.java | 4 +- .../db/exception/query/OutOfTTLException.java | 8 +- .../iotdb/db/exception/query/PathException.java | 38 - .../query/UnSupportedFillTypeException.java | 2 +- .../apache/iotdb/db/index/common/IndexType.java | 10 +- .../iotdb/db/localconfignode/LocalConfigNode.java | 335 +---- .../db/localconfignode/LocalDataPartitionInfo.java | 4 +- .../localconfignode/LocalDataPartitionTable.java | 2 +- .../localconfignode/LocalSchemaPartitionTable.java | 2 +- .../iotdb/db/metadata/LocalSchemaProcessor.java | 306 +--- .../apache/iotdb/db/metadata/MetadataConstant.java | 3 +- .../db/metadata/cache/DataNodeSchemaCache.java | 47 +- .../iotdb/db/metadata/cache/SchemaCacheEntry.java | 12 +- .../idtable/AppendOnlyDiskSchemaManager.java | 3 +- .../apache/iotdb/db/metadata/idtable/IDTable.java | 3 +- .../db/metadata/idtable/IDTableHashmapImpl.java | 136 +- .../iotdb/db/metadata/idtable/IDTableManager.java | 14 +- .../db/metadata/idtable/entry/DeviceEntry.java | 37 +- .../idtable/entry/InsertMeasurementMNode.java | 53 - .../iotdb/db/metadata/logfile/MLogWriter.java | 43 +- .../org/apache/iotdb/db/metadata/mnode/IMNode.java | 15 - .../iotdb/db/metadata/mnode/IMeasurementMNode.java | 5 - .../iotdb/db/metadata/mnode/InternalMNode.java | 32 - .../org/apache/iotdb/db/metadata/mnode/MNode.java | 29 - .../apache/iotdb/db/metadata/mnode/MNodeType.java | 2 +- .../iotdb/db/metadata/mnode/MeasurementMNode.java | 28 - .../db/metadata/mnode/StorageGroupEntityMNode.java | 4 +- .../iotdb/db/metadata/mtree/ConfigMTree.java | 70 +- .../iotdb/db/metadata/mtree/IMTreeBelowSG.java | 80 +- .../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 794 +++------- .../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 601 +------- .../mtree/snapshot/MemMTreeSnapshotUtil.java | 4 +- .../db/metadata/mtree/store/CachedMTreeStore.java | 29 +- .../mtree/store/disk/schemafile/ISchemaFile.java | 9 +- .../store/disk/schemafile/MockSchemaFile.java | 7 +- .../mtree/store/disk/schemafile/RecordUtils.java | 25 +- .../mtree/store/disk/schemafile/SchemaFile.java | 86 +- .../store/disk/schemafile/WrappedSegment.java | 2 +- .../db/metadata/mtree/traverser/Traverser.java | 53 +- .../mtree/traverser/collector/MNodeCollector.java | 2 +- .../traverser/collector/StorageGroupCollector.java | 2 +- .../traverser/counter/StorageGroupCounter.java | 2 +- .../plan/schemaregion/SchemaRegionPlanType.java | 6 +- .../plan/schemaregion/SchemaRegionPlanVisitor.java | 15 - .../impl/ActivateTemplatePlanImpl.java | 44 - .../impl/DeactivateTemplatePlanImpl.java | 6 + .../impl/PreDeactivateTemplatePlanImpl.java | 6 + .../RollbackPreDeactivateTemplatePlanImpl.java | 6 + .../impl/SchemaRegionPlanDeserializer.java | 44 +- .../schemaregion/impl/SchemaRegionPlanFactory.java | 29 +- .../impl/SchemaRegionPlanSerializer.java | 49 - .../impl/SchemaRegionPlanTxtSerializer.java | 29 - .../schemaregion/impl/SetTemplatePlanImpl.java | 69 - .../schemaregion/impl/UnsetTemplatePlanImpl.java | 69 - .../schemaregion/write/IActivateTemplatePlan.java | 42 - .../plan/schemaregion/write/ISetTemplatePlan.java | 45 - .../schemaregion/write/IUnsetTemplatePlan.java | 45 - .../db/metadata/schemaregion/ISchemaRegion.java | 33 +- .../db/metadata/schemaregion/SchemaEngine.java | 6 +- .../schemaregion/SchemaRegionMemoryImpl.java | 497 +------ .../schemaregion/SchemaRegionSchemaFileImpl.java | 807 +++------- .../metadata/schemaregion/SchemaRegionUtils.java | 48 - .../storagegroup/IStorageGroupSchemaManager.java | 82 +- .../storagegroup/StorageGroupSchemaManager.java | 8 +- .../apache/iotdb/db/metadata/tag/TagManager.java | 58 +- .../metadata/template/ClusterTemplateManager.java | 31 +- .../db/metadata/template/TemplateLogReader.java | 46 - .../db/metadata/template/TemplateLogWriter.java | 64 - .../db/metadata/template/TemplateManager.java | 384 ----- .../db/metadata/upgrade/MetadataUpgrader.java | 437 ------ .../iotdb/db/metadata/utils/MetaFormatUtils.java | 2 +- .../apache/iotdb/db/metadata/utils/MetaUtils.java | 24 +- .../db/mpp/common/header/ColumnHeaderConstant.java | 367 ++--- .../db/mpp/common/header/DatasetHeaderFactory.java | 4 + .../mpp/common/schematree/ClusterSchemaTree.java | 34 +- .../common/schematree/DeviceGroupSchemaTree.java | 7 +- .../db/mpp/common/schematree/ISchemaTree.java | 13 +- .../common/schematree/node/SchemaEntityNode.java | 1 + .../iotdb/db/mpp/execution/QueryStateMachine.java | 9 + .../iotdb/db/mpp/execution/StateMachine.java | 3 - .../db/mpp/execution/exchange/ISourceHandle.java | 8 + .../db/mpp/execution/exchange/LocalSinkHandle.java | 23 +- .../mpp/execution/exchange/LocalSourceHandle.java | 33 +- .../execution/exchange/MPPDataExchangeManager.java | 34 +- .../exchange/MPPDataExchangeServiceMetrics.java | 2 +- ...MppDataExchangeServiceThriftHandlerMetrics.java | 2 +- .../mpp/execution/exchange/SharedTsBlockQueue.java | 25 +- .../db/mpp/execution/exchange/SinkHandle.java | 98 +- .../db/mpp/execution/exchange/SourceHandle.java | 21 +- .../execution/executor/RegionWriteExecutor.java | 32 +- .../fragment/FragmentInstanceContext.java | 19 +- .../fragment/FragmentInstanceExecution.java | 11 +- .../fragment/FragmentInstanceFailureInfo.java | 212 +++ .../execution/fragment/FragmentInstanceInfo.java | 21 +- .../fragment/FragmentInstanceManager.java | 35 +- .../fragment/FragmentInstanceStateMachine.java | 14 +- .../operator/process/AbstractIntoOperator.java | 3 +- .../operator/process/DeviceViewOperator.java | 6 +- .../operator/process/FilterAndProjectOperator.java | 10 +- .../process/last/UpdateLastCacheOperator.java | 15 +- .../operator/schema/CountMergeOperator.java | 65 +- .../operator/schema/NodePathsCountOperator.java | 35 +- .../operator/schema/SchemaFetchMergeOperator.java | 2 +- .../operator/schema/SchemaFetchScanOperator.java | 2 +- .../schema/SchemaQueryOrderByHeatOperator.java | 133 +- .../schema/TimeSeriesSchemaScanOperator.java | 2 + .../execution/schedule/AbstractDriverThread.java | 2 +- .../db/mpp/execution/schedule/DriverScheduler.java | 2 +- .../schedule/FragmentInstanceAbortedException.java | 2 +- .../org/apache/iotdb/db/mpp/plan/Coordinator.java | 25 +- .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 5 + .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 122 +- .../mpp/plan/analyze/ClusterPartitionFetcher.java | 20 +- .../db/mpp/plan/analyze/ClusterSchemaFetcher.java | 209 ++- .../db/mpp/plan/analyze/ExpressionAnalyzer.java | 33 +- .../mpp/plan/analyze/ExpressionTypeAnalyzer.java | 12 +- .../db/mpp/plan/analyze/IPartitionFetcher.java | 8 +- .../plan/analyze/StandalonePartitionFetcher.java | 4 +- .../mpp/plan/analyze/StandaloneSchemaFetcher.java | 2 +- .../db/mpp/plan/analyze/cache/PartitionCache.java | 121 +- .../iotdb/db/mpp/plan/constant/StatementType.java | 2 - .../db/mpp/plan/execution/IQueryExecution.java | 4 + .../db/mpp/plan/execution/QueryExecution.java | 99 +- .../mpp/plan/execution/config/ConfigExecution.java | 10 + .../plan/execution/config/ConfigTaskVisitor.java | 5 +- .../config/executor/ClusterConfigTaskExecutor.java | 259 ++-- .../config/executor/IConfigTaskExecutor.java | 3 +- .../executor/StandaloneConfigTaskExecutor.java | 7 +- .../config/metadata/SetStorageGroupTask.java | 2 +- ...lusterTask.java => ShowClusterDetailsTask.java} | 78 +- .../execution/config/metadata/ShowClusterTask.java | 12 +- .../plan/execution/memory/MemorySourceHandle.java | 5 + .../iotdb/db/mpp/plan/expression/Expression.java | 5 + .../db/mpp/plan/expression/ExpressionType.java | 2 + .../db/mpp/plan/expression/leaf/NullOperand.java | 101 ++ .../plan/expression/unary/NegationExpression.java | 2 + .../visitor/ColumnTransformerVisitor.java | 17 + .../plan/expression/visitor/ExpressionVisitor.java | 5 + .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 69 +- .../db/mpp/plan/parser/StatementGenerator.java | 36 +- .../plan/planner/LocalExecutionPlanContext.java | 13 +- .../db/mpp/plan/planner/LocalExecutionPlanner.java | 9 +- .../db/mpp/plan/planner/LogicalPlanBuilder.java | 6 +- .../db/mpp/plan/planner/OperatorTreeGenerator.java | 7 +- .../plan/planner/distribution/SourceRewriter.java | 43 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 49 + .../plan/node/load/LoadSingleTsFileNode.java | 415 +----- .../planner/plan/node/load/LoadTsFileNode.java | 19 +- .../plan/node/load/LoadTsFilePieceNode.java | 13 +- .../write/CreateAlignedTimeSeriesNode.java | 6 +- .../metedata/write/CreateMultiTimeSeriesNode.java | 81 +- .../node/metedata/write/CreateTimeSeriesNode.java | 6 +- .../plan/node/metedata/write/MeasurementGroup.java | 96 +- .../plan/node/source/LastQueryScanNode.java | 6 +- .../planner/plan/node/write/InsertRowNode.java | 9 +- .../planner/plan/node/write/InsertRowsNode.java | 6 +- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 41 +- .../planner/plan/node/write/InsertTabletNode.java | 37 +- .../scheduler/AbstractFragInsStateTracker.java | 33 +- .../db/mpp/plan/scheduler/ClusterScheduler.java | 10 +- .../scheduler/FixedRateFragInsStateTracker.java | 61 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 4 +- .../plan/scheduler/IFragInstanceStateTracker.java | 6 + .../mpp/plan/scheduler/SimpleQueryTerminator.java | 30 +- .../db/mpp/plan/scheduler/StandaloneScheduler.java | 2 +- .../scheduler/load/LoadTsFileDispatcherImpl.java | 8 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 230 ++- .../crud/InsertMultiTabletsStatement.java | 2 +- .../plan/statement/crud/InsertRowStatement.java | 4 +- .../crud/InsertRowsOfOneDeviceStatement.java | 28 +- .../plan/statement/crud/InsertRowsStatement.java | 2 +- .../mpp/plan/statement/crud/InsertStatement.java | 1 - .../plan/statement/crud/InsertTabletStatement.java | 17 +- .../plan/statement/metadata/CountStatement.java | 2 +- .../statement/metadata/CreateTriggerStatement.java | 2 +- .../statement/metadata/DropTriggerStatement.java | 22 +- .../statement/metadata/ShowClusterStatement.java | 10 + .../statement/metadata/ShowDevicesStatement.java | 2 +- .../metadata/ShowStorageGroupStatement.java | 4 +- .../db/mpp/plan/statement/sys/FlushStatement.java | 2 +- .../sys/sync/CreatePipeSinkStatement.java | 7 +- ...izableRowRecordListBackedMultiColumnWindow.java | 12 + ...SerializableTVListBackedSingleColumnWindow.java | 12 + .../dag/adapter/EmptyRowIterator.java | 22 +- .../dag/column/ColumnTransformer.java | 31 + .../binary/CompareBinaryColumnTransformer.java | 7 +- .../binary/CompareEqualToColumnTransformer.java | 2 +- .../column/binary/LogicAndColumnTransformer.java | 30 + .../binary/LogicBinaryColumnTransformer.java | 24 - .../column/binary/LogicOrColumnTransformer.java | 30 + .../dag/column/leaf/NullColumnTransformer.java} | 20 +- .../ternary/CompareTernaryColumnTransformer.java | 17 +- .../dag/column/unary/InColumnTransformer.java | 48 +- .../column/unary/LogicNotColumnTransformer.java | 2 +- .../dag/column/unary/RegularColumnTransformer.java | 2 +- .../intermediate/ConstantIntermediateLayer.java | 3 +- .../MultiInputColumnIntermediateLayer.java | 21 +- ...InputColumnMultiReferenceIntermediateLayer.java | 20 +- ...nputColumnSingleReferenceIntermediateLayer.java | 20 +- .../transformation/dag/util/TransformUtils.java | 8 +- .../iotdb/db/protocol/influxdb/dto/IoTDBPoint.java | 14 - .../protocol/influxdb/handler/NewQueryHandler.java | 2 +- .../influxdb/meta/AbstractInfluxDBMetaManager.java | 2 +- .../influxdb/meta/NewInfluxDBMetaManager.java | 4 +- .../influxdb/meta/TagInfluxDBMetaManager.java | 2 +- .../db/protocol/influxdb/meta/TagInfoRecords.java | 24 - .../protocol/influxdb/util/QueryResultUtils.java | 11 +- .../protocol/rest/filter/AuthorizationFilter.java | 28 +- .../rest/handler/AuthorizationHandler.java | 4 +- .../db/protocol/rest/handler/ExceptionHandler.java | 10 +- .../rest/handler/ExecuteStatementHandler.java | 52 + .../protocol/rest/handler/QueryDataSetHandler.java | 37 +- .../rest/handler/StatementConstructionHandler.java | 3 +- .../protocol/rest/impl/GrafanaApiServiceImpl.java | 6 +- .../db/protocol/rest/impl/RestApiServiceImpl.java | 28 +- .../apache/iotdb/db/qp/constant/SQLConstant.java | 170 +-- .../apache/iotdb/db/qp/executor/IPlanExecutor.java | 79 - .../apache/iotdb/db/qp/executor/PlanExecutor.java | 1559 +------------------- .../db/qp/logical/crud/DeleteDataOperator.java | 15 +- .../iotdb/db/qp/logical/crud/InsertOperator.java | 31 +- .../db/qp/logical/sys/CreatePipeOperator.java | 12 +- .../db/qp/logical/sys/CreatePipeSinkOperator.java | 11 +- .../db/qp/logical/sys/CreateTriggerOperator.java | 92 -- .../db/qp/logical/sys/DeletePartitionOperator.java | 3 +- .../iotdb/db/qp/logical/sys/DropPipeOperator.java | 4 +- .../db/qp/logical/sys/DropPipeSinkOperator.java | 4 +- .../db/qp/logical/sys/DropTriggerOperator.java | 50 - .../iotdb/db/qp/logical/sys/ShowPipeOperator.java | 4 +- .../db/qp/logical/sys/ShowPipeSinkOperator.java | 4 +- .../qp/logical/sys/ShowPipeSinkTypeOperator.java | 4 +- .../iotdb/db/qp/logical/sys/StartPipeOperator.java | 4 +- .../db/qp/logical/sys/StartTriggerOperator.java | 50 - .../iotdb/db/qp/logical/sys/StopPipeOperator.java | 4 +- .../db/qp/logical/sys/StopTriggerOperator.java | 50 - .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 40 - .../db/qp/physical/crud/DeletePartitionPlan.java | 53 - .../iotdb/db/qp/physical/crud/DeletePlan.java | 221 --- .../qp/physical/crud/InsertMultiTabletsPlan.java | 453 ------ .../iotdb/db/qp/physical/crud/InsertPlan.java | 239 --- .../iotdb/db/qp/physical/crud/InsertRowPlan.java | 812 ---------- .../physical/crud/InsertRowsOfOneDevicePlan.java | 296 ---- .../iotdb/db/qp/physical/crud/InsertRowsPlan.java | 302 ---- .../db/qp/physical/crud/InsertTabletPlan.java | 895 ----------- .../db/qp/physical/sys/ActivateTemplatePlan.java | 4 +- .../db/qp/physical/sys/AppendTemplatePlan.java | 2 +- .../physical/sys/CreateAlignedTimeSeriesPlan.java | 8 +- .../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 6 +- .../iotdb/db/qp/physical/sys/CreatePipePlan.java | 110 -- .../db/qp/physical/sys/CreatePipeSinkPlan.java | 98 -- .../db/qp/physical/sys/CreateTemplatePlan.java | 7 +- .../db/qp/physical/sys/CreateTimeSeriesPlan.java | 6 +- .../db/qp/physical/sys/CreateTriggerPlan.java | 166 --- .../iotdb/db/qp/physical/sys/DropPipeSinkPlan.java | 45 - .../iotdb/db/qp/physical/sys/DropTriggerPlan.java | 102 -- .../apache/iotdb/db/qp/physical/sys/FlushPlan.java | 4 +- .../iotdb/db/qp/physical/sys/OperatePipePlan.java | 45 - .../iotdb/db/qp/physical/sys/SetTemplatePlan.java | 3 +- .../iotdb/db/qp/physical/sys/ShowPipePlan.java | 33 - .../iotdb/db/qp/physical/sys/ShowPipeSinkPlan.java | 33 - .../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 3 - .../iotdb/db/qp/physical/sys/StartTriggerPlan.java | 102 -- .../iotdb/db/qp/physical/sys/StopTriggerPlan.java | 102 -- .../db/qp/physical/sys/UnsetTemplatePlan.java | 3 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 115 +- .../db/query/control/QueryResourceManager.java | 6 +- .../iotdb/db/query/control/SessionManager.java | 4 +- .../iotdb/db/query/dataset/ShowDevicesDataSet.java | 4 +- .../db/query/dataset/ShowTimeSeriesResult.java | 17 +- .../db/query/dataset/ShowTimeseriesDataSet.java | 4 +- .../db/query/executor/AggregationExecutor.java | 2 +- .../iotdb/db/query/executor/LastQueryExecutor.java | 4 +- .../iotdb/db/query/pool/QueryTaskManager.java | 4 +- .../db/query/pool/RawQueryReadTaskPoolManager.java | 4 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 46 +- .../java/org/apache/iotdb/db/service/DataNode.java | 45 +- .../service/DataNodeInternalRPCServiceMetrics.java | 2 +- .../db/service/DataNodeServerCommandLine.java | 12 +- .../GetVersion.java} | 11 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 29 +- .../apache/iotdb/db/service/IoTDBShutdownHook.java | 13 +- .../org/apache/iotdb/db/service/MQTTService.java | 6 +- .../java/org/apache/iotdb/db/service/NewIoTDB.java | 38 +- .../apache/iotdb/db/service/RPCServiceMetrics.java | 2 +- .../iotdb/db/service/RegionMigrateService.java | 88 +- .../org/apache/iotdb/db/service/SettleService.java | 36 +- .../org/apache/iotdb/db/service/UpgradeSevice.java | 18 +- .../iotdb/db/service/metrics/ProcessMetrics.java | 10 +- .../InternalServiceThriftHandlerMetrics.java | 2 +- .../handler/RPCServiceThriftHandlerMetrics.java | 2 +- .../service/thrift/impl/ClientRPCServiceImpl.java | 138 +- .../impl/DataNodeInternalRPCServiceImpl.java | 61 +- .../service/thrift/impl/DataNodeRegionManager.java | 2 +- .../thrift/impl/NewInfluxDBServiceImpl.java | 2 +- .../db/service/thrift/impl/TSServiceImpl.java | 665 +-------- .../java/org/apache/iotdb/db/sync/SyncService.java | 79 +- .../db/sync/common/ClusterSyncInfoFetcher.java | 16 +- .../iotdb/db/sync/common/ISyncInfoFetcher.java | 3 - .../apache/iotdb/db/sync/common/LocalSyncInfo.java | 12 +- .../iotdb/db/sync/common/LocalSyncInfoFetcher.java | 12 +- .../iotdb/db/sync/datasource/PipeOpManager.java | 2 +- .../iotdb/db/sync/datasource/TsFileOpBlock.java | 5 +- .../db/sync/externalpipe/ExtPipePluginManager.java | 4 +- .../iotdb/db/sync/pipedata/DeletionPipeData.java | 14 +- .../apache/iotdb/db/sync/pipedata/PipeData.java | 35 +- .../iotdb/db/sync/pipedata/SchemaPipeData.java | 100 -- .../iotdb/db/sync/pipedata/TsFilePipeData.java | 32 +- .../sync/pipedata/queue/BufferedPipeDataQueue.java | 9 +- .../db/sync/receiver/load/DeletionLoader.java | 7 - .../iotdb/db/sync/receiver/load/SchemaLoader.java | 71 - .../iotdb/db/sync/receiver/load/TsFileLoader.java | 35 +- .../db/sync/sender/pipe/ExternalPipeSink.java | 20 - .../iotdb/db/sync/sender/pipe/TsFilePipe.java | 1 + .../db/sync/transport/client/IoTDBSyncClient.java | 29 +- .../db/sync/transport/client/SenderManager.java | 135 +- .../sync/transport/client/SyncClientFactory.java | 12 + .../db/sync/transport/server/ReceiverManager.java | 10 +- .../apache/iotdb/db/tools/IoTDBDataDirViewer.java | 4 +- .../apache/iotdb/db/tools/TsFileSelfCheckTool.java | 3 +- .../apache/iotdb/db/tools/TsFileSketchTool.java | 196 ++- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 10 +- .../java/org/apache/iotdb/db/tools/WalChecker.java | 12 +- .../db/tools/dataregion/DeviceMappingViewer.java | 63 - .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 6 +- .../db/tools/validate/TsFileValidationTool.java | 2 +- .../iotdb/db/trigger/executor/TriggerExecutor.java | 3 +- .../db/trigger/executor/TriggerFireVisitor.java | 8 +- .../trigger/service/TriggerInformationUpdater.java | 8 +- .../trigger/service/TriggerManagementService.java | 7 +- .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 8 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 27 +- .../java/org/apache/iotdb/db/utils/MemUtils.java | 54 - .../apache/iotdb/db/utils/QueryDataSetUtils.java | 12 +- .../apache/iotdb/db/utils/TimePartitionUtils.java | 16 +- .../apache/iotdb/db/utils/TypeInferenceUtils.java | 6 +- .../db/utils/datastructure/AlignedTVList.java | 4 +- .../iotdb/db/utils/datastructure/BinaryTVList.java | 17 +- .../db/utils/datastructure/BooleanTVList.java | 13 +- .../iotdb/db/utils/datastructure/DoubleTVList.java | 13 +- .../iotdb/db/utils/datastructure/FloatTVList.java | 13 +- .../iotdb/db/utils/datastructure/IntTVList.java | 13 +- .../iotdb/db/utils/datastructure/LongTVList.java | 13 +- .../iotdb/db/utils/datastructure/TVList.java | 20 +- .../apache/iotdb/db/utils/sync/SyncPipeUtil.java | 37 - .../runtime/WindowEvaluationTaskPoolManager.java | 5 +- .../java/org/apache/iotdb/db/wal/WALManager.java | 8 +- .../org/apache/iotdb/db/wal/buffer/WALEntry.java | 23 +- .../apache/iotdb/db/wal/buffer/WALEntryType.java | 7 +- .../apache/iotdb/db/wal/buffer/WALInfoEntry.java | 20 +- .../iotdb/db/wal/exception/WALException.java | 6 +- .../org/apache/iotdb/db/wal/node/IWALNode.java | 11 - .../org/apache/iotdb/db/wal/node/WALFakeNode.java | 19 - .../java/org/apache/iotdb/db/wal/node/WALNode.java | 46 +- .../iotdb/db/wal/recover/WALNodeRecoverTask.java | 2 +- .../iotdb/db/wal/recover/WALRecoverManager.java | 6 +- .../db/wal/recover/file/TsFilePlanRedoer.java | 103 +- .../file/UnsealedTsFileRecoverPerformer.java | 13 +- .../iotdb/db/auth/AuthorizerManagerTest.java | 4 +- .../org/apache/iotdb/db/auth/entity/UserTest.java | 4 +- .../engine/compaction/AbstractCompactionTest.java | 69 +- .../engine/compaction/CompactionSchedulerTest.java | 386 ++--- .../ReadPointCompactionPerformerTest.java | 158 +- .../cross/CrossSpaceCompactionExceptionTest.java | 3 +- .../compaction/cross/CrossSpaceCompactionTest.java | 20 +- .../cross/CrossSpaceCompactionValidationTest.java | 3 +- .../cross/RewriteCompactionFileSelectorTest.java | 14 +- .../RewriteCrossSpaceCompactionRecoverTest.java | 3 +- .../cross/RewriteCrossSpaceCompactionTest.java | 16 +- .../inner/InnerCompactionEmptyTsFileTest.java | 1 + .../inner/InnerCompactionMoreDataTest.java | 63 +- .../inner/InnerCompactionSchedulerTest.java | 13 +- .../compaction/inner/InnerCompactionTest.java | 20 +- .../compaction/inner/InnerSeqCompactionTest.java | 6 +- .../ReadChunkCompactionPerformerNoAlignedTest.java | 18 +- .../inner/ReadChunkCompactionPerformerOldTest.java | 2 +- .../SizeTieredCompactionSelectorTest.java | 4 +- .../inner/sizetiered/SizeTieredCompactionTest.java | 8 +- ...eCrossSpaceCompactionRecoverCompatibleTest.java | 3 +- .../SizeTieredCompactionRecoverCompatibleTest.java | 3 +- .../compaction/utils/CompactionCheckerUtils.java | 9 +- .../compaction/utils/CompactionConfigRestorer.java | 2 +- .../utils/MultiTsFileDeviceIteratorTest.java | 120 ++ .../db/engine/memtable/MemTableFlushTaskTest.java | 6 +- .../db/engine/memtable/MemTableTestUtils.java | 69 +- .../db/engine/memtable/PrimitiveMemTableTest.java | 65 +- .../db/engine/storagegroup/DataRegionTest.java | 182 ++- .../storagegroup/FileNodeManagerBenchmark.java | 132 -- .../storagegroup/IDTableLastFlushTimeMapTest.java | 129 ++ .../engine/storagegroup/LastFlushTimeMapTest.java | 223 +++ .../storagegroup/StorageGroupProcessorTest.java | 871 ----------- .../iotdb/db/engine/storagegroup/TTLTest.java | 175 +-- .../storagegroup/TimePartitionManagerTest.java | 129 ++ .../engine/storagegroup/TsFileProcessorTest.java | 476 ------ .../engine/storagegroup/TsFileProcessorV2Test.java | 12 +- .../dataregion/HashVirtualPartitionerTest.java | 63 - .../LocalConfigNodeMultiDataRegionTest.java | 4 - .../apache/iotdb/db/metadata/SchemaBasicTest.java | 1539 ++----------------- .../org/apache/iotdb/db/metadata/TemplateTest.java | 322 +--- .../db/metadata/cache/DataNodeSchemaCacheTest.java | 7 +- .../db/metadata/idtable/IDTableFlushTimeTest.java | 226 --- .../db/metadata/idtable/IDTableLogFileTest.java | 239 ++- .../db/metadata/idtable/IDTableRecoverTest.java | 354 ++--- .../iotdb/db/metadata/idtable/IDTableTest.java | 1479 ++++++++++--------- .../db/metadata/idtable/LastQueryWithIDTable.java | 285 ++-- .../iotdb/db/metadata/mtree/ConfigMTreeTest.java | 8 +- .../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 2 +- .../plan/SchemaRegionPlanCompatibilityTest.java | 57 +- .../schemaRegion/SchemaRegionBasicTest.java | 274 +++- .../schemaRegion/SchemaRegionSnapshotTest.java | 168 --- .../db/metadata/upgrade/MetadataUpgradeTest.java | 318 ---- .../common/schematree/ClusterSchemaTreeTest.java | 25 + .../db/mpp/execution/exchange/SinkHandleTest.java | 28 +- .../FragmentInstanceFailureInfoSerdeTest.java | 85 ++ .../mpp/execution/operator/OperatorMemoryTest.java | 11 +- .../operator/schema/CountMergeOperatorTest.java | 4 +- .../schema/SchemaQueryScanOperatorTest.java | 27 +- .../schedule/DefaultDriverSchedulerTest.java | 3 +- .../db/mpp/plan/StandaloneCoordinatorTest.java | 2 - .../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 10 +- .../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 2 +- .../mpp/plan/analyze/cache/PartitionCacheTest.java | 6 +- .../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 2 +- .../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 6 +- .../distribution/AggregationDistributionTest.java | 139 +- .../distribution/DistributionPlannerBasicTest.java | 126 +- .../distribution/NoDataRegionPlanningTest.java | 37 +- .../iotdb/db/mpp/plan/plan/distribution/Util.java | 492 ++++-- .../plan/node/write/WritePlanNodeSplitTest.java | 18 +- .../plan/scheduler/StandaloneSchedulerTest.java | 2 - .../db/protocol/rest/IoTDBRestServiceTest.java | 18 +- .../java/org/apache/iotdb/db/qp/PlannerTest.java | 71 +- .../iotdb/db/qp/logical/LogicalPlanSmallTest.java | 4 +- .../iotdb/db/qp/other/TSPlanContextAuthorTest.java | 2 +- .../iotdb/db/qp/physical/ConcatOptimizerTest.java | 138 -- .../qp/physical/InsertRowsOfOneDevicePlanTest.java | 75 - .../db/qp/physical/PhysicalPlanSerializeTest.java | 326 ---- .../iotdb/db/qp/physical/PhysicalPlanTest.java | 1484 ------------------- .../iotdb/db/qp/physical/SerializationTest.java | 151 -- .../iotdb/db/qp/physical/SerializedSizeTest.java | 130 -- .../iotdb/db/query/dataset/ListDataSetTest.java | 12 +- .../db/query/dataset/ShowTimeSeriesResultTest.java | 4 +- .../iotdb/db/query/dataset/SingleDataSetTest.java | 199 ++- .../org/apache/iotdb/db/service/DaemonTest.java} | 21 +- .../db/sync/persistence/LocalSyncInfoTest.java | 16 +- .../sync/pipedata/BufferedPipeDataQueueTest.java | 491 +++--- .../iotdb/db/sync/pipedata/PipeDataTest.java | 11 - .../org/apache/iotdb/db/tools/MLogParserTest.java | 41 +- .../iotdb/db/tools/TsFileAndModSettleToolTest.java | 390 +++-- .../org/apache/iotdb/db/tools/WalCheckerTest.java | 28 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 37 +- .../org/apache/iotdb/db/utils/MemUtilsTest.java | 35 +- .../iotdb/db/utils/TsFileRewriteToolTest.java | 27 +- .../iotdb/db/utils/TypeInferenceUtilsTest.java | 2 + .../db/utils/datastructure/PrecisionTest.java | 143 -- .../db/utils/writelog/LogWriterReaderTest.java | 50 +- .../db/utils/writelog/MultiFileLogReaderTest.java | 15 +- .../org/apache/iotdb/db/wal/WALManagerTest.java | 52 +- .../db/wal/allocation/ElasticStrategyTest.java | 16 +- .../db/wal/allocation/FirstCreateStrategyTest.java | 18 +- .../db/wal/allocation/RoundRobinStrategyTest.java | 16 +- .../iotdb/db/wal/buffer/WALBufferCommonTest.java | 68 +- .../org/apache/iotdb/db/wal/io/WALFileTest.java | 119 +- .../apache/iotdb/db/wal/node/WALFakeNodeTest.java | 22 +- .../org/apache/iotdb/db/wal/node/WALNodeTest.java | 90 +- .../db/wal/recover/WALRecoverManagerTest.java | 116 +- .../file/SealedTsFileRecoverPerformerTest.java | 32 +- .../db/wal/recover/file/TsFilePlanRedoerTest.java | 244 +-- .../file/UnsealedTsFileRecoverPerformerTest.java | 60 +- .../iotdb-common.properties} | 14 +- ...{iotdb-metric.yml => iotdb-datanode-metric.yml} | 2 +- .../datanode1conf/iotdb-datanode.properties | 7 +- .../resources/datanode1conf/iotdb-rest.properties | 45 +- .../iotdb-common.properties} | 14 +- ...{iotdb-metric.yml => iotdb-datanode-metric.yml} | 2 +- .../datanode2conf/iotdb-datanode.properties | 7 +- .../resources/datanode2conf/iotdb-rest.properties | 43 - .../iotdb-common.properties} | 14 +- ...{iotdb-metric.yml => iotdb-datanode-metric.yml} | 2 +- .../datanode3conf/iotdb-datanode.properties | 11 +- .../resources/datanode3conf/iotdb-rest.properties | 45 +- .../src/test/resources/iotdb-datanode.properties | 2 - server/src/test/resources/iotdb-rest.properties | 3 + .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 40 +- .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 8 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 305 ++-- .../java/org/apache/iotdb/session/ISession.java | 13 + .../java/org/apache/iotdb/session/Session.java | 24 +- .../apache/iotdb/session/SessionConnection.java | 9 +- .../org/apache/iotdb/session/SessionDataSet.java | 12 +- .../org/apache/iotdb/session/pool/SessionPool.java | 63 + .../src/test/resources/iotdb-datanode.properties | 11 +- session/src/test/resources/iotdb-engine.properties | 20 - session/src/test/resources/iotdb-rest.properties | 3 + site/src/main/.vuepress/config.js | 64 +- .../src/test/resources/iotdb-datanode.properties | 11 +- .../src/test/resources/iotdb-engine.properties | 21 - .../apache/iotdb/spark/db/EnvironmentUtils.java | 20 +- .../iotdb/spark/tsfile/qp/common/SQLConstant.java | 123 +- .../apache/iotdb/spark/tsfile/DefaultSource.scala | 10 +- .../iotdb/spark/tsfile/NarrowConverter.scala | 181 ++- .../org/apache/iotdb/spark/tsfile/TSFileSuit.scala | 30 +- test/e2e/cases/cli/res/init.sql | 4 +- .../test/java/org/apache/iotdb/db/sync/SyncIT.java | 6 +- thrift-commons/src/main/thrift/common.thrift | 2 +- .../src/main/thrift/confignode.thrift | 53 +- thrift/src/main/thrift/client.thrift | 4 +- thrift/src/main/thrift/datanode.thrift | 12 +- .../iotdb/tsfile/file/header/ChunkHeader.java | 21 + .../iotdb/tsfile/file/header/PageHeader.java | 10 +- .../file/metadata/enums/CompressionType.java | 21 +- .../iotdb/tsfile/read/TsFileDeviceIterator.java | 34 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 130 +- .../org/apache/iotdb/tsfile/read/common/Path.java | 9 +- .../iotdb/tsfile/read/common/block/TsBlock.java | 5 +- .../read/common/block/column/NullColumn.java | 87 +- .../read/controller/MetadataQuerierByFileImpl.java | 3 +- .../apache/iotdb/tsfile/utils/FilePathUtils.java | 4 +- .../iotdb/tsfile/utils/TsFileGeneratorUtils.java | 2 +- .../apache/iotdb/tsfile/write/TsFileWriter.java | 4 +- .../write/chunk/AlignedChunkGroupWriterImpl.java | 6 +- .../tsfile/write/chunk/AlignedChunkWriterImpl.java | 4 + .../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 1 + .../tsfile/write/chunk/IChunkGroupWriter.java | 4 +- .../iotdb/tsfile/write/chunk/TimeChunkWriter.java | 1 + .../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 12 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 30 +- .../write/writer/TsFileIOWriterEndFileTest.java | 49 + .../write/writer/tsmiterator/DiskTSMIterator.java | 7 +- .../write/writer/tsmiterator/TSMIterator.java | 9 +- .../tsfile/common/block/TsBlockSerdeTest.java | 4 +- ...easurementChunkMetadataListMapIteratorTest.java | 2 +- .../iotdb/tsfile/read/ReadInPartitionTest.java | 22 +- .../org/apache/iotdb/tsfile/read/ReadTest.java | 54 +- .../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 55 +- .../tsfile/read/TimeSeriesMetadataReadTest.java | 4 +- .../apache/iotdb/tsfile/read/TsFileReaderTest.java | 126 +- .../tsfile/read/controller/ChunkLoaderTest.java | 2 +- .../controller/IMetadataQuerierByFileImplTest.java | 16 +- .../read/filter/IExpressionOptimizerTest.java | 30 +- .../read/query/executor/QueryExecutorTest.java | 32 +- .../read/query/timegenerator/ReadWriteTest.java | 8 +- .../query/timegenerator/ReaderByTimestampTest.java | 2 +- .../timegenerator/TimeGeneratorReadEmptyTest.java | 8 +- .../timegenerator/TimeGeneratorReadWriteTest.java | 8 +- .../query/timegenerator/TimeGeneratorTest.java | 6 +- .../iotdb/tsfile/read/reader/ChunkReaderTest.java | 2 +- .../tsfile/read/reader/FakedTimeGenerator.java | 9 +- .../iotdb/tsfile/read/reader/ReaderTest.java | 8 +- .../tsfile/write/DefaultSchemaTemplateTest.java | 2 +- .../tsfile/write/MetadataIndexConstructorTest.java | 6 +- ...SameMeasurementsWithDifferentDataTypesTest.java | 4 +- .../iotdb/tsfile/write/TsFileReadWriteTest.java | 4 +- .../iotdb/tsfile/write/TsFileWriteApiTest.java | 264 ++++ .../iotdb/tsfile/write/TsFileWriterTest.java | 10 +- .../org/apache/iotdb/tsfile/write/WriteTest.java | 2 +- .../write/writer/ForceAppendTsFileWriterTest.java | 4 +- .../write/writer/RestorableTsFileIOWriterTest.java | 36 +- .../writer/TsFileIOWriterMemoryControlTest.java | 11 +- .../org/apache/iotdb/udf/api/access/RowWindow.java | 3 +- .../iotdb/udf/api/collector/PointCollector.java | 4 +- .../strategy/SessionTimeWindowAccessStrategy.java | 10 +- zeppelin-interpreter/IoTDB-Zeppelin-Demo.zpln | 2 +- .../zeppelin/iotdb/IoTDBInterpreterTest.java | 35 +- 1295 files changed, 26310 insertions(+), 44486 deletions(-) diff --cc server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java index 814fd22deb,43515a61f2..10408a935c --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java @@@ -130,10 -128,8 +131,9 @@@ import java.util.Map import java.util.Optional; import static org.apache.iotdb.db.service.basic.ServiceProvider.AUDIT_LOGGER; - import static org.apache.iotdb.db.service.basic.ServiceProvider.CONFIG; import static org.apache.iotdb.db.service.basic.ServiceProvider.CURRENT_RPC_VERSION; import static org.apache.iotdb.db.service.basic.ServiceProvider.QUERY_FREQUENCY_RECORDER; +import static org.apache.iotdb.db.service.basic.ServiceProvider.SLOW_SQL_LOGGER; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onNPEOrUnexpectedException; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; @@@ -1856,17 -1772,9 +1890,17 @@@ public class ClientRPCServiceImpl imple return resp; } + private TSFetchWindowBatchResp createTSFetchWindowBatchResp(DatasetHeader header) { + TSFetchWindowBatchResp resp = RpcUtils.getTSFetchWindowBatchResp(TSStatusCode.SUCCESS_STATUS); + resp.setColumnNameList(header.getRespColumns()); + resp.setColumnTypeList(header.getRespDataTypeList()); + resp.setColumnNameIndexMap(header.getColumnNameIndexMap()); + return resp; + } + private TSStatus getNotLoggedInStatus() { return RpcUtils.getStatus( - TSStatusCode.NOT_LOGIN_ERROR, + TSStatusCode.NOT_LOGIN, "Log in failed. Either you are not authorized or the session has timed out."); } diff --cc server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java index b0d26a7b0d,e0575880ea..629e6efa74 --- a/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java @@@ -398,259 -401,9 +402,259 @@@ public class QueryDataSetUtils } rowCount += positionCount; } - return res; + return new Pair<>(res, !queryExecution.hasNextResult()); } + public static List<List<ByteBuffer>> convertTsBlocksToWindowBatch( + IQueryExecution queryExecution, List<Integer> samplingIndexes) throws IoTDBException { + List<List<ByteBuffer>> sortedWindowBatch = new ArrayList<>(); + + while (true) { + Optional<ByteBuffer> optionalByteBuffer = queryExecution.getByteBufferBatchResult(); + if (!optionalByteBuffer.isPresent()) { + break; + } + + List<ByteBuffer> res = new ArrayList<>(); + ByteBuffer byteBuffer = optionalByteBuffer.get(); + byteBuffer.mark(); + int valueColumnCount = byteBuffer.getInt(); + for (int i = 0; i < valueColumnCount; i++) { + byteBuffer.get(); + } + int positionCount = byteBuffer.getInt(); + byteBuffer.reset(); + if (positionCount != 0) { + res.add(byteBuffer); + } + + sortedWindowBatch.add(res); + } + + List<List<ByteBuffer>> windowBatch = new ArrayList<>(sortedWindowBatch.size()); + List<Integer> sortedSamplingIndexes = + samplingIndexes.stream().sorted().collect(Collectors.toList()); + + for (Integer samplingIndex : samplingIndexes) { + int mapIndex = sortedSamplingIndexes.indexOf(samplingIndex); + windowBatch.add(sortedWindowBatch.get(mapIndex)); + } + return windowBatch; + } + + public static List<TSQueryDataSet> convertTsBlocksToWindowBatchDataSetList( + IQueryExecution queryExecution, List<Integer> samplingIndexes) + throws IoTDBException, IOException { + List<TSQueryDataSet> sortedWindowBatch = new ArrayList<>(); + + int columnNum = queryExecution.getOutputValueColumnCount(); + // one time column and each value column has an actual value buffer and a bitmap value to + // indicate whether it is a null + int columnNumWithTime = columnNum * 2 + 1; + + while (true) { + Optional<TsBlock> optionalTsBlock = queryExecution.getBatchResult(); + if (!optionalTsBlock.isPresent()) { + break; + } + TsBlock tsBlock = optionalTsBlock.get(); + if (tsBlock.isEmpty()) { + continue; + } + + TSQueryDataSet tsQueryDataSet = new TSQueryDataSet(); + + DataOutputStream[] dataOutputStreams = new DataOutputStream[columnNumWithTime]; + ByteArrayOutputStream[] byteArrayOutputStreams = new ByteArrayOutputStream[columnNumWithTime]; + for (int i = 0; i < columnNumWithTime; i++) { + byteArrayOutputStreams[i] = new ByteArrayOutputStream(); + dataOutputStreams[i] = new DataOutputStream(byteArrayOutputStreams[i]); + } + + int rowCount = 0; + int[] valueOccupation = new int[columnNum]; + + // used to record a bitmap for every 8 points + int[] bitmaps = new int[columnNum]; + + int currentCount = tsBlock.getPositionCount(); + // serialize time column + for (int i = 0; i < currentCount; i++) { + // use columnOutput to write byte array + dataOutputStreams[0].writeLong(tsBlock.getTimeByIndex(i)); + } + + // serialize each value column and its bitmap + for (int k = 0; k < columnNum; k++) { + // get DataOutputStream for current value column and its bitmap + DataOutputStream dataOutputStream = dataOutputStreams[2 * k + 1]; + DataOutputStream dataBitmapOutputStream = dataOutputStreams[2 * (k + 1)]; + + Column column = tsBlock.getColumn(k); + TSDataType type = column.getDataType(); + switch (type) { + case INT32: + for (int i = 0; i < currentCount; i++) { + rowCount++; + if (column.isNull(i)) { + bitmaps[k] = bitmaps[k] << 1; + } else { + bitmaps[k] = (bitmaps[k] << 1) | FLAG; + dataOutputStream.writeInt(column.getInt(i)); + valueOccupation[k] += 4; + } + if (rowCount != 0 && rowCount % 8 == 0) { + dataBitmapOutputStream.writeByte(bitmaps[k]); + // we should clear the bitmap every 8 points + bitmaps[k] = 0; + } + } + break; + case INT64: + for (int i = 0; i < currentCount; i++) { + rowCount++; + if (column.isNull(i)) { + bitmaps[k] = bitmaps[k] << 1; + } else { + bitmaps[k] = (bitmaps[k] << 1) | FLAG; + dataOutputStream.writeLong(column.getLong(i)); + valueOccupation[k] += 8; + } + if (rowCount != 0 && rowCount % 8 == 0) { + dataBitmapOutputStream.writeByte(bitmaps[k]); + // we should clear the bitmap every 8 points + bitmaps[k] = 0; + } + } + break; + case FLOAT: + for (int i = 0; i < currentCount; i++) { + rowCount++; + if (column.isNull(i)) { + bitmaps[k] = bitmaps[k] << 1; + } else { + bitmaps[k] = (bitmaps[k] << 1) | FLAG; + dataOutputStream.writeFloat(column.getFloat(i)); + valueOccupation[k] += 4; + } + if (rowCount != 0 && rowCount % 8 == 0) { + dataBitmapOutputStream.writeByte(bitmaps[k]); + // we should clear the bitmap every 8 points + bitmaps[k] = 0; + } + } + break; + case DOUBLE: + for (int i = 0; i < currentCount; i++) { + rowCount++; + if (column.isNull(i)) { + bitmaps[k] = bitmaps[k] << 1; + } else { + bitmaps[k] = (bitmaps[k] << 1) | FLAG; + dataOutputStream.writeDouble(column.getDouble(i)); + valueOccupation[k] += 8; + } + if (rowCount != 0 && rowCount % 8 == 0) { + dataBitmapOutputStream.writeByte(bitmaps[k]); + // we should clear the bitmap every 8 points + bitmaps[k] = 0; + } + } + break; + case BOOLEAN: + for (int i = 0; i < currentCount; i++) { + rowCount++; + if (column.isNull(i)) { + bitmaps[k] = bitmaps[k] << 1; + } else { + bitmaps[k] = (bitmaps[k] << 1) | FLAG; + dataOutputStream.writeBoolean(column.getBoolean(i)); + valueOccupation[k] += 1; + } + if (rowCount != 0 && rowCount % 8 == 0) { + dataBitmapOutputStream.writeByte(bitmaps[k]); + // we should clear the bitmap every 8 points + bitmaps[k] = 0; + } + } + break; + case TEXT: + for (int i = 0; i < currentCount; i++) { + rowCount++; + if (column.isNull(i)) { + bitmaps[k] = bitmaps[k] << 1; + } else { + bitmaps[k] = (bitmaps[k] << 1) | FLAG; + Binary binary = column.getBinary(i); + dataOutputStream.writeInt(binary.getLength()); + dataOutputStream.write(binary.getValues()); + valueOccupation[k] = valueOccupation[k] + 4 + binary.getLength(); + } + if (rowCount != 0 && rowCount % 8 == 0) { + dataBitmapOutputStream.writeByte(bitmaps[k]); + // we should clear the bitmap every 8 points + bitmaps[k] = 0; + } + } + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", type)); + } + if (k != columnNum - 1) { + rowCount -= currentCount; + } + } + + // feed the remaining bitmap + int remaining = rowCount % 8; + for (int k = 0; k < columnNum; k++) { + if (remaining != 0) { + DataOutputStream dataBitmapOutputStream = dataOutputStreams[2 * (k + 1)]; + dataBitmapOutputStream.writeByte(bitmaps[k] << (8 - remaining)); + } + } + + // calculate the time buffer size + int timeOccupation = rowCount * 8; + ByteBuffer timeBuffer = ByteBuffer.allocate(timeOccupation); + timeBuffer.put(byteArrayOutputStreams[0].toByteArray()); + timeBuffer.flip(); + tsQueryDataSet.setTime(timeBuffer); + + // calculate the bitmap buffer size + int bitmapOccupation = (rowCount + 7) / 8; + + List<ByteBuffer> bitmapList = new LinkedList<>(); + List<ByteBuffer> valueList = new LinkedList<>(); + for (int i = 1; i < byteArrayOutputStreams.length; i += 2) { + ByteBuffer valueBuffer = ByteBuffer.allocate(valueOccupation[(i - 1) / 2]); + valueBuffer.put(byteArrayOutputStreams[i].toByteArray()); + valueBuffer.flip(); + valueList.add(valueBuffer); + + ByteBuffer bitmapBuffer = ByteBuffer.allocate(bitmapOccupation); + bitmapBuffer.put(byteArrayOutputStreams[i + 1].toByteArray()); + bitmapBuffer.flip(); + bitmapList.add(bitmapBuffer); + } + tsQueryDataSet.setBitmapList(bitmapList); + tsQueryDataSet.setValueList(valueList); + + sortedWindowBatch.add(tsQueryDataSet); + } + + List<TSQueryDataSet> windowBatch = new ArrayList<>(sortedWindowBatch.size()); + List<Integer> sortedSamplingIndexes = + samplingIndexes.stream().sorted().collect(Collectors.toList()); + + for (Integer samplingIndex : samplingIndexes) { + int mapIndex = sortedSamplingIndexes.indexOf(samplingIndex); + windowBatch.add(sortedWindowBatch.get(mapIndex)); + } + return windowBatch; + } + public static long[] readTimesFromBuffer(ByteBuffer buffer, int size) { long[] times = new long[size]; for (int i = 0; i < size; i++) { diff --cc session/src/main/java/org/apache/iotdb/session/SessionConnection.java index f0a11bcecc,bd2ca927bc..a934c0fefa --- a/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@@ -482,52 -480,10 +484,53 @@@ public class SessionConnection client, sessionId, tsExecuteStatementResp.queryResult, - tsExecuteStatementResp.isIgnoreTimeStamp()); + tsExecuteStatementResp.isIgnoreTimeStamp(), + tsExecuteStatementResp.moreData); } + public List<SessionDataSet> fetchWindowBatch( + List<String> queryPaths, + String functionName, + long startTime, + long endTime, + long interval, + long slidingStep, + List<Integer> indexes) + throws StatementExecutionException { + TSFetchWindowBatchReq req = + new TSFetchWindowBatchReq( + sessionId, + statementId, + queryPaths, + new TGroupByTimeParameter(startTime, endTime, interval, slidingStep, indexes)); + if (functionName != null) { + req.setFunctionName(functionName); + } + + TSFetchWindowBatchResp resp; + try { + resp = client.fetchWindowBatchV2(req); + RpcUtils.verifySuccess(resp.getStatus()); + } catch (TException e) { + throw new StatementExecutionException(""); + } + + List<SessionDataSet> windowSet = new ArrayList<>(); + for (List<ByteBuffer> window : resp.getWindowBatch()) { + SessionDataSet sessionDataSet = + new SessionDataSet( + resp.columnNameList, + resp.columnTypeList, + resp.columnNameIndexMap, + -1, + statementId, + sessionId, + window); + windowSet.add(sessionDataSet); + } + return windowSet; + } + protected void insertRecord(TSInsertRecordReq request) throws IoTDBConnectionException, StatementExecutionException, RedirectException { request.setSessionId(sessionId);
