This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch expr in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1a4365c591101eaf69dcb3196c8f4869231649af Merge: abcf608 8759fa4 Author: jt <[email protected]> AuthorDate: Tue Jan 4 10:47:46 2022 +0800 Merge branch 'master' into expr # Conflicts: # cluster/src/main/java/org/apache/iotdb/cluster/ClusterFileFlushPolicy.java # cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/ElectionHandler.java # cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java # cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java .github/workflows/grafana-plugin.yml | 47 + Jenkinsfile | 2 +- LICENSE-binary | 4 +- RELEASE_NOTES.md | 2 +- .../main/java/org/apache/iotdb/tool/ImportCsv.java | 18 +- cluster/src/assembly/cluster.xml | 4 + .../cluster/log/applier/AsyncDataLogApplier.java | 2 +- .../iotdb/cluster/log/applier/BaseApplier.java | 2 +- .../iotdb/cluster/log/applier/DataLogApplier.java | 6 +- .../log/manage/PartitionedSnapshotLogManager.java | 15 + .../log/sequencing/SynchronousSequencer.java | 8 + .../apache/iotdb/cluster/metadata/CMManager.java | 76 +- .../partition/slot/SlotTimePartitionFilter.java | 2 +- .../iotdb/cluster/query/ClusterPlanExecutor.java | 30 +- .../iotdb/cluster/query/ClusterPlanRouter.java | 14 +- .../iotdb/cluster/query/ClusterQueryRouter.java | 19 - .../iotdb/cluster/query/LocalQueryExecutor.java | 26 +- .../ClusterGroupByFillNoVFilterDataSet.java | 54 - .../groupby/ClusterGroupByFillVFilterDataSet.java | 67 - .../iotdb/cluster/server/ClusterRPCService.java | 8 +- .../server/handlers/caller/ElectionHandler.java | 17 + .../handlers/caller/ShowTimeSeriesHandler.java | 20 +- .../cluster/server/member/DataGroupMember.java | 2 +- .../iotdb/cluster/server/member/RaftMember.java | 12 +- .../apache/iotdb/cluster/utils/PartitionUtils.java | 2 +- .../cluster/utils/nodetool/ClusterMonitor.java | 83 + .../org/apache/iotdb/cluster/common/IoTDBTest.java | 2 +- .../org/apache/iotdb/cluster/common/TestUtils.java | 4 +- .../cluster/log/applier/DataLogApplierTest.java | 26 +- .../cluster/log/logtypes/SerializeLogTest.java | 3 +- .../cluster/log/snapshot/FileSnapshotTest.java | 14 +- .../log/snapshot/PartitionedSnapshotTest.java | 4 +- .../cluster/log/snapshot/PullSnapshotTaskTest.java | 4 +- .../mult/AssignPathManagedMergeReaderTest.java | 3 +- .../reader/mult/RemoteMultSeriesReaderTest.java | 3 +- .../iotdb/cluster/server/member/BaseMember.java | 2 + .../cluster/server/member/DataGroupMemberTest.java | 19 +- .../cluster/server/member/MetaGroupMemberTest.java | 5 +- .../resources/node1conf/iotdb-engine.properties | 1 + .../src/test/resources/node1conf}/iotdb-metric.yml | 22 +- .../resources/node2conf/iotdb-engine.properties | 1 + .../src/test/resources/node2conf}/iotdb-metric.yml | 22 +- .../resources/node3conf/iotdb-engine.properties | 1 + .../src/test/resources/node3conf}/iotdb-metric.yml | 22 +- code-coverage/pom.xml | 1 + compile-tools/thrift/pom.xml | 33 - distribution/pom.xml | 14 +- distribution/src/assembly/all.xml | 4 +- .../{grafana.xml => grafana-connector.xml} | 8 +- .../assembly/{grafana.xml => grafana-plugin.xml} | 26 +- docker/ReadMe.md | 2 +- docs/Download/README.md | 45 +- docs/SystemDesign/StorageEngine/Recover.md | 2 +- docs/SystemDesign/StorageEngine/StorageEngine.md | 2 +- docs/SystemDesign/TsFile/Format.md | 1 - docs/UserGuide/API/Programming-Cpp-Native-API.md | 62 +- docs/UserGuide/Appendix/Config-Manual.md | 38 +- docs/UserGuide/Cluster/Cluster-Setup-Example.md | 107 +- docs/UserGuide/Cluster/Cluster-Setup.md | 4 +- docs/UserGuide/Data-Concept/Encoding.md | 4 +- .../UserGuide/Data-Modeling/VirtualStorageGroup.md | 55 + .../{Grafana.md => Grafana Connector.md} | 22 +- .../DML-Data-Manipulation-Language.md | 492 +- docs/UserGuide/Library-UDF/Data-Quality.md | 520 + docs/UserGuide/Library-UDF/Data-Repair.md | 349 + docs/UserGuide/Library-UDF/Get-Started.md | 61 + docs/UserGuide/System-Tools/Metric-Tool.md | 288 +- docs/zh/Download/README.md | 45 +- docs/zh/SystemDesign/StorageEngine/Recover.md | 4 +- .../zh/SystemDesign/StorageEngine/StorageEngine.md | 2 +- docs/zh/SystemDesign/TsFile/Format.md | 1 - .../zh/UserGuide/API/Programming-Cpp-Native-API.md | 80 +- docs/zh/UserGuide/Appendix/Config-Manual.md | 37 + docs/zh/UserGuide/Cluster/Cluster-Setup-Example.md | 108 +- docs/zh/UserGuide/Cluster/Cluster-Setup.md | 4 +- docs/zh/UserGuide/Data-Concept/Encoding.md | 3 +- .../UserGuide/Data-Modeling/VirtualStorageGroup.md | 53 + .../{Grafana.md => Grafana Connector.md} | 20 +- .../Ecosystem Integration/Grafana Plugin.md | 250 + .../DML-Data-Manipulation-Language.md | 494 +- docs/zh/UserGuide/Library-UDF/Data-Quality.md | 509 + docs/zh/UserGuide/Library-UDF/Data-Repair.md | 341 + docs/zh/UserGuide/Library-UDF/Get-Started.md | 58 + docs/zh/UserGuide/System-Tools/Metric-Tool.md | 280 +- .../img/add_data_source.png | Bin {grafana => grafana-connector}/img/add_graph.png | Bin .../img/edit_data_source.png | Bin {grafana => grafana-connector}/pom.xml | 4 +- {grafana => grafana-connector}/readme.md | 22 +- {grafana => grafana-connector}/readme_zh.md | 20 +- .../web/grafana/TsfileWebDemoApplication.java | 0 .../apache/iotdb/web/grafana/bean/TimeValues.java | 0 .../iotdb/web/grafana/conf/MyConfiguration.java | 0 .../controller/DatabaseConnectController.java | 0 .../org/apache/iotdb/web/grafana/dao/BasicDao.java | 0 .../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 0 .../grafana/service/DatabaseConnectService.java | 0 .../service/impl/DatabaseConnectServiceImpl.java | 0 .../src/main/resources/application.properties | 0 .../web/grafana/dao/impl/BasicDaoImplTest.java | 0 grafana-plugin/.gitignore | 32 + .../.prettierrc.js | 23 +- grafana-plugin/CHANGELOG.md | 24 + LICENSE-binary => grafana-plugin/LICENSE | 105 +- grafana-plugin/README.md | 63 + .../jest.config.js | 25 +- grafana-plugin/package.json | 31 + grafana-plugin/pom.xml | 96 + grafana-plugin/src/ConfigEditor.tsx | 101 + grafana-plugin/src/QueryEditor.tsx | 114 + .../src/componments/ControlValue.tsx | 40 +- .../src/componments/Form.tsx | 39 +- grafana-plugin/src/componments/FromValue.tsx | 110 + grafana-plugin/src/componments/SelectValue.tsx | 109 + .../src/componments/WhereValue.tsx | 40 +- grafana-plugin/src/datasource.ts | 164 + .../src/functions.ts | 22 +- grafana-plugin/src/img/logo.svg | 271 + .../src/module.ts | 28 +- grafana-plugin/src/plugin.json | 47 + .../src/types.ts | 41 +- grafana-plugin/tsconfig.json | 9 + grafana-plugin/yarn.lock | 14427 +++++++++++++++++++ .../org/apache/iotdb/influxdb/IoTDBInfluxDB.java | 18 +- .../iotdb/integration/env/ClusterEnvBase.java | 6 +- .../iotdb/integration/env/ClusterEnvConfig.java | 72 +- .../apache/iotdb/integration/env/ClusterNode.java | 6 +- .../org/apache/iotdb/itbase/env/BaseConfig.java | 24 +- .../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 5 +- .../iotdb/db/integration/IoTDBDeletionIT.java | 6 +- .../iotdb/db/integration/IoTDBGroupByFillIT.java | 275 +- .../db/integration/IoTDBGroupByFillMixPathsIT.java | 759 +- .../integration/IoTDBGroupByFillWithRangeIT.java | 14 +- .../db/integration/IoTDBGroupByMonthFillIT.java | 55 +- .../apache/iotdb/db/integration/IoTDBMergeIT.java | 24 +- .../iotdb/db/integration/IoTDBMultiDeviceIT.java | 75 +- .../db/integration/IoTDBNewTsFileCompactionIT.java | 6 +- .../db/integration/IoTDBPartialInsertionIT.java | 69 + .../db/integration/IoTDBQueryWithIDTableIT.java | 289 + .../iotdb/db/integration/IoTDBRestartIT.java | 55 - .../iotdb/db/integration/IoTDBSimpleQueryIT.java | 38 +- .../aggregation/IoTDBAggregationByLevelIT.java | 12 +- .../aligned/IoTDBAggregationGroupByLevelIT.java | 12 +- .../IoTDBAggregationWithoutValueFilterIT.java | 18 +- .../integration/aligned/IoTDBAlignByDevice2IT.java | 68 + .../integration/aligned/IoTDBAlignByDeviceIT.java | 1349 ++ .../db/integration/aligned/IoTDBDeletionIT.java | 88 +- .../aligned/IoTDBGroupByFillQueryBigDataIT.java | 192 + .../aligned/IoTDBGroupByFillQueryIT.java | 989 ++ .../db/integration/env/StandaloneEnvConfig.java | 17 + .../apache/iotdb/session/IoTDBSessionSimpleIT.java | 2 +- library-udf/pom.xml | 159 + .../dropwizard/DropwizardMetricManagerTest.java | 1 + .../main/assembly/resources/conf/iotdb-metric.yml | 1 + .../org/apache/iotdb/metrics/MetricService.java | 6 +- .../apache/iotdb/metrics/config/MetricConfig.java | 3 +- .../metrics/config/MetricConfigDescriptor.java | 12 +- .../iotdb/metrics/utils/PredefinedMetric.java | 3 +- .../micrometer/MicrometerMetricManager.java | 11 + .../reporter/MicrometerPrometheusReporter.java | 27 +- .../micrometer/MicrometerMetricManagerTest.java | 1 + openapi/src/main/openapi3/iotdb-rest.yaml | 61 + pom.xml | 9 +- .../resources/conf/iotdb-engine.properties | 31 +- .../db/concurrent/IoTDBThreadPoolFactory.java | 10 +- .../org/apache/iotdb/db/concurrent/ThreadName.java | 1 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 76 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 35 + .../org/apache/iotdb/db/engine/StorageEngine.java | 218 +- .../apache/iotdb/db/engine/cache/ChunkCache.java | 16 + .../db/engine/cache/TimeSeriesMetadataCache.java | 46 +- .../engine/compaction/CompactionTaskManager.java | 51 + .../inplace/manage/CrossSpaceMergeResource.java | 25 +- .../cross/inplace/task/CrossSpaceMergeTask.java | 19 +- .../cross/inplace/task/MergeMultiChunkTask.java | 46 +- .../InnerSpaceCompactionExceptionHandler.java | 6 +- .../SizeTieredCompactionRecoverTask.java | 2 +- .../inner/sizetiered/SizeTieredCompactionTask.java | 80 +- .../compaction/task/AbstractCompactionTask.java | 18 + .../compaction/task/CompactionRecoverTask.java | 2 +- .../apache/iotdb/db/engine/flush/FlushManager.java | 28 + .../iotdb/db/engine/flush/MemTableFlushTask.java | 21 +- .../iotdb/db/engine/flush/TsFileFlushPolicy.java | 11 +- .../iotdb/db/engine/memtable/AbstractMemTable.java | 120 +- .../apache/iotdb/db/engine/memtable/IMemTable.java | 17 +- .../db/engine/memtable/PrimitiveMemTable.java | 6 +- .../engine/storagegroup/ILastFlushTimeManager.java | 89 + .../engine/storagegroup/LastFlushTimeManager.java | 268 + .../db/engine/storagegroup/StorageGroupInfo.java | 18 +- .../db/engine/storagegroup/TsFileProcessor.java | 82 +- .../engine/storagegroup/TsFileProcessorInfo.java | 23 + .../db/engine/storagegroup/TsFileResource.java | 4 +- ...ssor.java => VirtualStorageGroupProcessor.java} | 262 +- ...eGroupManager.java => StorageGroupManager.java} | 167 +- .../service/TriggerRegistrationService.java | 29 +- .../trigger/sink/local/LocalIoTDBHandler.java | 2 +- .../org/apache/iotdb/db/metadata/MManager.java | 104 +- .../idtable/AppendOnlyDiskSchemaManager.java | 163 + .../apache/iotdb/db/metadata/idtable/IDTable.java | 181 + .../db/metadata/idtable/IDTableHashmapImpl.java | 463 + .../iotdb/db/metadata/idtable/IDTableManager.java | 105 + .../idtable/IDiskSchemaManager.java} | 43 +- .../db/metadata/idtable/entry/DeviceEntry.java | 81 + .../db/metadata/idtable/entry/DeviceIDFactory.java | 87 + .../db/metadata/idtable/entry/DiskSchemaEntry.java | 107 + .../iotdb/db/metadata/idtable/entry/IDeviceID.java | 15 +- .../idtable/entry/InsertMeasurementMNode.java | 267 + .../idtable/entry/PlainDeviceID.java} | 41 +- .../db/metadata/idtable/entry/SHA256DeviceID.java | 148 + .../db/metadata/idtable/entry/SchemaEntry.java | 193 + .../idtable/entry/TimeseriesID.java} | 36 +- .../org/apache/iotdb/db/metadata/mtree/MTree.java | 7 +- .../apache/iotdb/db/metadata/path/AlignedPath.java | 10 +- .../iotdb/db/metadata/path/MeasurementPath.java | 12 +- .../apache/iotdb/db/metadata/path/PartialPath.java | 113 +- .../apache/iotdb/db/metadata/tag/TagManager.java | 4 +- .../org/apache/iotdb/db/monitor/StatMonitor.java | 8 +- .../iotdb/db/protocol/influxdb/dto/IoTDBPoint.java | 26 +- .../protocol/influxdb/input/InfluxLineParser.java | 6 + .../rest/handler/AuthorizationHandler.java | 5 +- .../protocol/rest/handler/QueryDataSetHandler.java | 36 +- .../rest/handler/RequestValidationHandler.java | 9 + .../protocol/rest/impl/GrafanaApiServiceImpl.java | 169 + .../db/protocol/rest/impl/PingApiServiceImpl.java | 1 + .../db/protocol/rest/impl/RestApiServiceImpl.java | 50 +- .../apache/iotdb/db/qp/executor/IPlanExecutor.java | 2 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 33 +- .../qp/logical/crud/GroupByFillQueryOperator.java | 2 +- .../iotdb/db/qp/logical/crud/QueryOperator.java | 151 +- .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 17 +- .../iotdb/db/qp/physical/crud/AggregationPlan.java | 50 + .../db/qp/physical/crud/AlignByDevicePlan.java | 140 +- .../db/qp/physical/crud/DeletePartitionPlan.java | 2 +- .../iotdb/db/qp/physical/crud/DeletePlan.java | 8 +- .../db/qp/physical/crud/GroupByTimeFillPlan.java | 59 + .../iotdb/db/qp/physical/crud/GroupByTimePlan.java | 26 +- .../db/qp/physical/crud/InsertMultiTabletPlan.java | 14 +- .../iotdb/db/qp/physical/crud/InsertPlan.java | 43 +- .../iotdb/db/qp/physical/crud/InsertRowPlan.java | 36 +- .../physical/crud/InsertRowsOfOneDevicePlan.java | 18 +- .../iotdb/db/qp/physical/crud/InsertRowsPlan.java | 4 +- .../db/qp/physical/crud/InsertTabletPlan.java | 20 +- .../iotdb/db/qp/physical/crud/LastQueryPlan.java | 18 + .../iotdb/db/qp/physical/crud/MeasurementInfo.java | 37 +- .../iotdb/db/qp/physical/crud/QueryIndexPlan.java | 12 + .../iotdb/db/qp/physical/crud/QueryPlan.java | 77 +- .../db/qp/physical/crud/RawDataQueryPlan.java | 8 +- .../iotdb/db/qp/physical/crud/SelectIntoPlan.java | 4 +- .../apache/iotdb/db/qp/physical/crud/UDAFPlan.java | 13 + .../apache/iotdb/db/qp/physical/crud/UDTFPlan.java | 13 + .../db/qp/physical/sys/ActivateTemplatePlan.java | 4 +- .../db/qp/physical/sys/AlterTimeSeriesPlan.java | 4 +- .../db/qp/physical/sys/AppendTemplatePlan.java | 6 +- .../iotdb/db/qp/physical/sys/AuthorPlan.java | 4 +- .../qp/physical/sys/AutoCreateDeviceMNodePlan.java | 8 +- .../iotdb/db/qp/physical/sys/ChangeAliasPlan.java | 4 +- .../db/qp/physical/sys/ChangeTagOffsetPlan.java | 4 +- .../iotdb/db/qp/physical/sys/ClearCachePlan.java | 2 +- .../physical/sys/CreateAlignedTimeSeriesPlan.java | 4 +- .../qp/physical/sys/CreateContinuousQueryPlan.java | 7 +- .../db/qp/physical/sys/CreateFunctionPlan.java | 4 +- .../iotdb/db/qp/physical/sys/CreateIndexPlan.java | 4 +- .../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 2 +- .../db/qp/physical/sys/CreateSnapshotPlan.java | 2 +- .../db/qp/physical/sys/CreateTemplatePlan.java | 6 +- .../db/qp/physical/sys/CreateTimeSeriesPlan.java | 4 +- .../db/qp/physical/sys/CreateTriggerPlan.java | 4 +- .../iotdb/db/qp/physical/sys/DataAuthPlan.java | 4 +- .../db/qp/physical/sys/DeleteStorageGroupPlan.java | 4 +- .../db/qp/physical/sys/DeleteTimeSeriesPlan.java | 6 +- .../qp/physical/sys/DropContinuousQueryPlan.java | 4 +- .../iotdb/db/qp/physical/sys/DropFunctionPlan.java | 4 +- .../iotdb/db/qp/physical/sys/DropIndexPlan.java | 4 +- .../iotdb/db/qp/physical/sys/DropTriggerPlan.java | 4 +- .../apache/iotdb/db/qp/physical/sys/DummyPlan.java | 2 +- .../apache/iotdb/db/qp/physical/sys/FlushPlan.java | 6 +- .../iotdb/db/qp/physical/sys/KillQueryPlan.java | 2 +- .../db/qp/physical/sys/LoadConfigurationPlan.java | 6 +- .../iotdb/db/qp/physical/sys/LoadDataPlan.java | 2 +- .../apache/iotdb/db/qp/physical/sys/LogPlan.java | 6 +- .../apache/iotdb/db/qp/physical/sys/MNodePlan.java | 12 +- .../db/qp/physical/sys/MeasurementMNodePlan.java | 4 +- .../apache/iotdb/db/qp/physical/sys/MergePlan.java | 4 +- .../iotdb/db/qp/physical/sys/OperateFilePlan.java | 6 +- .../db/qp/physical/sys/PruneTemplatePlan.java | 4 +- .../db/qp/physical/sys/SetStorageGroupPlan.java | 4 +- .../db/qp/physical/sys/SetSystemModePlan.java | 4 +- .../iotdb/db/qp/physical/sys/SetTTLPlan.java | 4 +- .../iotdb/db/qp/physical/sys/SetTemplatePlan.java | 4 +- .../iotdb/db/qp/physical/sys/SettlePlan.java | 4 +- .../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 4 +- .../iotdb/db/qp/physical/sys/StartTriggerPlan.java | 4 +- .../iotdb/db/qp/physical/sys/StopTriggerPlan.java | 4 +- .../db/qp/physical/sys/StorageGroupMNodePlan.java | 4 +- .../iotdb/db/qp/physical/sys/TracingPlan.java | 2 +- .../db/qp/physical/sys/UnsetTemplatePlan.java | 4 +- .../apache/iotdb/db/qp/utils/DatetimeUtils.java | 19 + .../apache/iotdb/db/qp/utils/WildcardsRemover.java | 13 +- .../db/query/control/tracing/TracingManager.java | 4 +- .../db/query/dataset/AlignByDeviceDataSet.java | 121 +- .../db/query/dataset/NonAlignEngineDataSet.java | 4 +- .../dataset/RawQueryDataSetWithoutValueFilter.java | 9 +- .../db/query/dataset/ShowTimeSeriesResult.java | 13 + .../dataset/groupby/GroupByEngineDataSet.java | 2 +- .../query/dataset/groupby/GroupByFillDataSet.java | 328 + .../dataset/groupby/GroupByFillEngineDataSet.java | 424 - .../groupby/GroupByFillWithValueFilterDataSet.java | 451 - .../GroupByFillWithoutValueFilterDataSet.java | 267 - .../groupby/GroupByWithValueFilterDataSet.java | 4 +- .../groupby/GroupByWithoutValueFilterDataSet.java | 4 +- .../db/query/executor/AggregationExecutor.java | 6 +- .../iotdb/db/query/executor/FillQueryExecutor.java | 4 +- .../iotdb/db/query/executor/LastQueryExecutor.java | 64 +- .../iotdb/db/query/executor/QueryRouter.java | 44 +- .../db/query/executor/RawDataQueryExecutor.java | 14 +- .../apache/iotdb/db/query/executor/fill/IFill.java | 22 +- .../db/query/executor/fill/LastPointReader.java | 3 +- .../query/expression/unary/FunctionExpression.java | 3 +- ...yTaskPoolManager.java => QueryTaskManager.java} | 18 +- .../db/query/pool/RawQueryReadTaskPoolManager.java | 120 + .../iotdb/db/query/reader/series/SeriesReader.java | 5 +- .../query/timegenerator/ServerTimeGenerator.java | 4 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 10 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 2 +- .../org/apache/iotdb/db/service/RPCService.java | 8 +- .../db/service/basic/BasicServiceProvider.java | 4 + .../apache/iotdb/db/service/metrics/Metric.java | 25 +- .../iotdb/db/service/metrics/MetricsService.java | 112 +- .../org/apache/iotdb/db/service/metrics/Tag.java | 15 +- .../db/service/thrift/ProcessorWithMetrics.java | 70 + .../service/thrift/impl/InfluxDBServiceImpl.java | 3 +- .../db/service/thrift/impl/TSServiceImpl.java | 629 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 16 + .../java/org/apache/iotdb/db/utils/FileUtils.java | 24 + .../org/apache/iotdb/db/utils/SchemaUtils.java | 11 +- .../iotdb/db/writelog/recover/LogReplayer.java | 44 +- .../writelog/recover/TsFileRecoverPerformer.java | 8 +- .../recover/SizeTieredCompactionRecoverTest.java | 4 +- .../db/engine/memtable/MemTableFlushTaskTest.java | 3 +- .../db/engine/memtable/MemTableTestUtils.java | 6 +- .../db/engine/memtable/MemtableBenchmark.java | 7 +- .../db/engine/memtable/PrimitiveMemTableTest.java | 31 +- .../engine/modification/DeletionFileNodeTest.java | 10 +- .../storagegroup/FileNodeManagerBenchmark.java | 3 +- .../storagegroup/StorageGroupProcessorTest.java | 4 +- .../iotdb/db/engine/storagegroup/TTLTest.java | 56 +- .../apache/iotdb/db/metadata/PartialPathTest.java | 55 +- .../db/metadata/idtable/IDTableLogFileTest.java | 160 + .../idtable/IDTableResourceControlTest.java | 167 + .../db/metadata/idtable/IDTableRestartTest.java | 205 + .../iotdb/db/metadata/idtable/IDTableTest.java | 666 + .../db/metadata/idtable/InsertWithIDTableTest.java | 758 + .../db/metadata/idtable/LastQueryWithIDTable.java | 180 + .../QueryAlignedTimeseriesWithIDTableTest.java | 373 + .../db/metadata/idtable/QueryWithIDTableTest.java | 370 + .../db/metadata/idtable/entry/DeviceIDTest.java} | 44 +- .../db/metadata/idtable/entry/SchemaEntryTest.java | 74 + .../metadata/idtable/trigger_example/Counter.java | 96 + .../db/protocol/influxdb/dto/IoTDBPointTest.java | 39 +- .../db/protocol/rest/GrafanaApiServiceIT.java | 293 + .../db/query/dataset/ShowTimeSeriesResultTest.java | 2 + .../db/sync/receiver/load/FileLoaderTest.java | 12 +- .../recover/SyncReceiverLogAnalyzerTest.java | 4 +- .../iotdb/db/writelog/recover/LogReplayerTest.java | 3 +- .../recover/RecoverResourceFromReaderTest.java | 2 +- .../db/writelog/recover/SeqTsFileRecoverTest.java | 6 +- .../writelog/recover/UnseqTsFileRecoverTest.java | 2 +- .../java/org/apache/iotdb/session/Session.java | 27 +- .../session/IoTDBSessionVectorABDeviceIT.java | 27 +- .../iotdb/session/IoTDBSessionVectorInsertIT.java | 4 - site/src/main/.vuepress/config.js | 21 +- .../public/img/contributor-avatar/ericpai.jpg | Bin 0 -> 265364 bytes .../public/img/contributor-avatar/qhl.jpg | Bin 0 -> 365541 bytes .../theme/global-components/Contributor.vue | 15 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 172 +- .../tsfile/read/filter/factory/FilterFactory.java | 4 + .../iotdb/tsfile/read/filter/operator/Regexp.java | 21 +- .../org/apache/iotdb/tsfile/utils/TsFileUtils.java | 38 + .../tsfile/encoding/decoder/IntRleDecoderTest.java | 7 +- .../regular/RegularDataEncoderLongTest.java | 4 +- .../iotdb/tsfile/read/GetAllDevicesTest.java | 2 +- .../tsfile/read/filter/FilterSerializeTest.java | 1 + .../apache/iotdb/tsfile/utils/FileGenerator.java | 32 +- .../apache/iotdb/tsfile/utils/TsFileUtilsTest.java | 72 + .../tsfile/write/MetadataIndexConstructorTest.java | 61 +- 385 files changed, 33273 insertions(+), 5119 deletions(-) diff --cc cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java index f7e6704,0000000..1c2390a mode 100644,000000..100644 --- a/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java @@@ -1,109 -1,0 +1,117 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.cluster.log.sequencing; + +import org.apache.iotdb.cluster.log.Log; +import org.apache.iotdb.cluster.log.LogDispatcher.SendLogRequest; +import org.apache.iotdb.cluster.log.VotingLog; ++import org.apache.iotdb.cluster.log.logtypes.PhysicalPlanLog; +import org.apache.iotdb.cluster.log.manage.RaftLogManager; +import org.apache.iotdb.cluster.rpc.thrift.AppendEntryRequest; +import org.apache.iotdb.cluster.server.member.RaftMember; +import org.apache.iotdb.cluster.server.monitor.Timer; +import org.apache.iotdb.cluster.server.monitor.Timer.Statistic; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; ++import org.apache.iotdb.db.qp.physical.PhysicalPlan; ++import org.apache.iotdb.db.qp.physical.sys.LogPlan; + +/** + * SynchronizedSequencer performs sequencing by taking the monitor of a LogManager within the caller + * thread. + */ +public class SynchronousSequencer implements LogSequencer { + + private RaftMember member; + private RaftLogManager logManager; + + public SynchronousSequencer(RaftMember member, RaftLogManager logManager) { + this.member = member; + this.logManager = logManager; + } + + @Override + public SendLogRequest sequence(Log log) { + SendLogRequest sendLogRequest; + + long startTime = + Statistic.RAFT_SENDER_COMPETE_LOG_MANAGER_BEFORE_APPEND_V2.getOperationStartTime(); + synchronized (logManager) { + Statistic.RAFT_SENDER_COMPETE_LOG_MANAGER_BEFORE_APPEND_V2.calOperationCostTimeFromStart( + startTime); + + log.setCurrLogTerm(member.getTerm().get()); + log.setCurrLogIndex(logManager.getLastLogIndex() + 1); + ++ if (log instanceof PhysicalPlanLog) { ++ PhysicalPlanLog physicalPlanLog = (PhysicalPlanLog) log; ++ physicalPlanLog.getPlan().setIndex(log.getCurrLogIndex()); ++ } ++ + startTime = Timer.Statistic.RAFT_SENDER_APPEND_LOG_V2.getOperationStartTime(); + + // logDispatcher will serialize log, and set log size, and we will use the size after it + logManager.append(log); + Timer.Statistic.RAFT_SENDER_APPEND_LOG_V2.calOperationCostTimeFromStart(startTime); + + startTime = Statistic.RAFT_SENDER_BUILD_LOG_REQUEST.getOperationStartTime(); + sendLogRequest = buildSendLogRequest(log); + Statistic.RAFT_SENDER_BUILD_LOG_REQUEST.calOperationCostTimeFromStart(startTime); + + startTime = Statistic.RAFT_SENDER_OFFER_LOG.getOperationStartTime(); + log.setCreateTime(System.nanoTime()); + member.getVotingLogList().insert(sendLogRequest.getVotingLog()); + member.getLogDispatcher().offer(sendLogRequest); + Statistic.RAFT_SENDER_OFFER_LOG.calOperationCostTimeFromStart(startTime); + } + return sendLogRequest; + } + + @Override + public void setLogManager(RaftLogManager logManager) { + this.logManager = logManager; + } + + private SendLogRequest buildSendLogRequest(Log log) { + VotingLog votingLog = member.buildVotingLog(log); + AtomicBoolean leaderShipStale = new AtomicBoolean(false); + AtomicLong newLeaderTerm = new AtomicLong(member.getTerm().get()); + + long startTime = Statistic.RAFT_SENDER_BUILD_APPEND_REQUEST.getOperationStartTime(); + AppendEntryRequest appendEntryRequest = member.buildAppendEntryRequest(log, false); + Statistic.RAFT_SENDER_BUILD_APPEND_REQUEST.calOperationCostTimeFromStart(startTime); + + return new SendLogRequest( + votingLog, + leaderShipStale, + newLeaderTerm, + appendEntryRequest, + member.getAllNodes().size() / 2); + } + + public static class Factory implements LogSequencerFactory { + + @Override + public LogSequencer create(RaftMember member, RaftLogManager logManager) { + return new SynchronousSequencer(member, logManager); + } + } +} diff --cc cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/ElectionHandler.java index cbb5a08,229673b..fee8449 --- a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/ElectionHandler.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/ElectionHandler.java @@@ -96,8 -101,9 +101,9 @@@ public class ElectionHandler implement // the election is valid electionValid.set(true); terminated.set(true); - raftMember.getTerm().notifyAll(); raftMember.onElectionWins(); + raftMember.getTerm().notifyAll(); + result = "win"; logger.info("{}: Election {} is won", memberName, currTerm); } // still need more votes diff --cc cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java index f961cd8,1e71fed..d71f5e3 --- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java @@@ -1128,8 -992,10 +1129,11 @@@ public abstract class RaftMember implem } // assign term and index to the new log and append it + VotingLog votingLog; synchronized (logManager) { + if (!(plan instanceof LogPlan)) { + plan.setIndex(logManager.getLastLogIndex() + 1); + } log.setCurrLogTerm(getTerm().get()); log.setCurrLogIndex(logManager.getLastLogIndex() + 1); logManager.append(log); diff --cc library-udf/pom.xml index 0000000,3ca979a..02f2cd2 mode 000000,100644..100644 --- a/library-udf/pom.xml +++ b/library-udf/pom.xml @@@ -1,0 -1,163 +1,159 @@@ + <?xml version="1.0" encoding="UTF-8"?> + <!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> ++<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>iotdb-parent</artifactId> + <groupId>org.apache.iotdb</groupId> + <version>0.13.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> - + <artifactId>library-udf</artifactId> + <version>2.0.0</version> - + <properties> + <maven.compiler.source>8</maven.compiler.source> + <maven.compiler.target>8</maven.compiler.target> + </properties> + <dependencies> + <dependency> + <groupId>org.eclipse.collections</groupId> + <artifactId>eclipse-collections</artifactId> + <version>10.4.0</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-server</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>tsfile</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-session</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-math3</artifactId> + <version>3.5</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <version>3.9</version> + </dependency> + <!-- TimeStampRepair --> + <dependency> + <groupId>gov.nist.math</groupId> + <artifactId>jama</artifactId> + <version>1.0.3</version> + </dependency> + <!-- LOF --> + <dependency> + <groupId>com.github.chen0040</groupId> + <artifactId>java-local-outlier-factor</artifactId> + <version>1.0.4</version> + </dependency> + <!-- FFT, IFFT, LowPass, HighPass --> + <dependency> + <groupId>com.github.wendykierp</groupId> + <artifactId>JTransforms</artifactId> + <version>3.1</version> + </dependency> + <dependency> + <groupId>org.jetbrains</groupId> + <artifactId>annotations</artifactId> + <version>RELEASE</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>2.8.5</version> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <version>5.8.2</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>3.1.1</version> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <finalName>${project.build.finalName}-jar-with-dependencies</finalName> + <appendAssemblyId>false</appendAssemblyId> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.3</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <version>2.4.3</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>3.0.0-M5</version> + <dependencies> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>5.8.2</version> + </dependency> + </dependencies> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.6.0</version> + <configuration> + <classpathScope>test</classpathScope> + </configuration> + </plugin> + </plugins> + </build> -</project> ++</project> diff --cc server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java index 356c572,52244ba..0b4770b --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java @@@ -101,11 -100,8 +101,11 @@@ public abstract class PhysicalPlan // login username, corresponding to cli/session login user info private String loginUserName; -- // a bridge from a cluster raft log to a physical plan -- protected long index; ++ // a bridge from a cluster raft log to a physical plan, -1 means unset ++ protected long index = -1; + // if set, only the associated leader can execute the plan to guarantee serializability and other + // leaders should return a LEADER_CHANGED response + protected long targetedTerm = -1; private boolean debug; diff --cc server/src/main/java/org/apache/iotdb/db/qp/physical/sys/DummyPlan.java index 67f7637,0000000..42e701f mode 100644,000000..100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/DummyPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/DummyPlan.java @@@ -1,95 -1,0 +1,95 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.qp.physical.sys; + +import org.apache.iotdb.db.exception.metadata.IllegalPathException; +import org.apache.iotdb.db.metadata.path.PartialPath; +import org.apache.iotdb.db.qp.logical.Operator.OperatorType; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; + +public class DummyPlan extends PhysicalPlan { + + private byte[] workload; + private boolean needForward; + + public DummyPlan() { - super(false, OperatorType.EMPTY); ++ super(OperatorType.EMPTY); + } + + @Override + public List<PartialPath> getPaths() { + return Collections.emptyList(); + } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + stream.write((byte) PhysicalPlanType.DUMMY.ordinal()); + stream.writeInt(workload == null ? 0 : workload.length); + if (workload != null) { + stream.write(workload); + } + stream.write(needForward ? 1 : 0); + } + + @Override + public void serialize(ByteBuffer buffer) { + buffer.put((byte) PhysicalPlanType.DUMMY.ordinal()); + buffer.putInt(workload == null ? 0 : workload.length); + if (workload != null) { + buffer.put(workload); + } + buffer.put(needForward ? (byte) 1 : 0); + } + + @Override + public void deserialize(ByteBuffer buffer) throws IllegalPathException { + int size = buffer.getInt(); + workload = new byte[size]; + buffer.get(workload); + needForward = buffer.get() == 1; + } + + public void setWorkload(byte[] workload) { + this.workload = workload; + } + + public boolean isNeedForward() { + return needForward; + } + + public void setNeedForward(boolean needForward) { + this.needForward = needForward; + } + + public byte[] getWorkload() { + return workload; + } + + @Override + public String toString() { + return "ExprPlan"; + } +} diff --cc server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java index 9452cba,0d06348..99f49eb --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java @@@ -473,27 -529,34 +531,35 @@@ public class TSServiceImpl extends Basi return RpcUtils.getTSExecuteStatementResp(getNotLoggedInStatus()); } + long startTime = System.currentTimeMillis(); PhysicalPlan physicalPlan = processor.parseSQLToPhysicalPlan(statement, sessionManager.getZoneId(req.getSessionId())); + physicalPlan.setTargetedTerm(req.latestTerm); - return physicalPlan.isQuery() - ? internalExecuteQueryStatement( - statement, - req.statementId, - physicalPlan, - req.fetchSize, - req.timeout, - req.getSessionId(), - req.isEnableRedirectQuery(), - req.isJdbcQuery()) - : executeUpdateStatement( - statement, - req.statementId, - physicalPlan, - req.fetchSize, - req.timeout, - req.getSessionId()); + if (physicalPlan.isQuery()) { + Future<TSExecuteStatementResp> resp = + QueryTaskManager.getInstance() + .submit( + new QueryTask( + physicalPlan, + startTime, + req.sessionId, + req.statement, + req.statementId, + req.timeout, + req.fetchSize, + req.jdbcQuery, + req.enableRedirectQuery)); + return resp.get(); + } else { + return executeUpdateStatement( + statement, + req.statementId, + physicalPlan, + req.fetchSize, + req.timeout, + req.getSessionId()); + } } catch (InterruptedException e) { LOGGER.error(INFO_INTERRUPT_ERROR, req, e); Thread.currentThread().interrupt();
