This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch DoubleWrite in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 87c5b3db53ec4b285f575b61068b1ddca2ffe59f Merge: c96ef52 3b14591 Author: JackieTien97 <[email protected]> AuthorDate: Mon May 17 21:40:41 2021 +0800 fix conflicts .github/workflows/client-go.yml | 3 + .github/workflows/client.yml | 5 +- .github/workflows/e2e.yml | 3 + .github/workflows/main-unix.yml | 3 + .github/workflows/main-win.yml | 3 + .github/workflows/sonar-coveralls.yml | 3 + .../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 6 +- cli/pom.xml | 2 +- .../org/apache/iotdb/tool/AbstractCsvTool.java | 40 +-- .../main/java/org/apache/iotdb/tool/ExportCsv.java | 2 +- .../main/java/org/apache/iotdb/tool/ImportCsv.java | 56 ++-- client-cpp/src/main/Session.cpp | 11 +- client-cpp/src/main/Session.h | 16 +- cluster/pom.xml | 12 +- .../resources/conf/iotdb-cluster.properties | 60 ++-- .../apache/iotdb/cluster/config/ClusterConfig.java | 20 +- .../iotdb/cluster/config/ClusterDescriptor.java | 5 - .../iotdb/cluster/log/applier/BaseApplier.java | 36 ++- .../iotdb/cluster/log/catchup/CatchUpTask.java | 48 +-- .../iotdb/cluster/log/catchup/LogCatchUpTask.java | 4 +- .../cluster/log/manage/CommittedEntryManager.java | 12 +- .../serializable/SyncLogDequeSerializer.java | 4 +- .../iotdb/cluster/server/DataClusterServer.java | 6 +- .../iotdb/cluster/server/MetaClusterServer.java | 6 +- .../iotdb/cluster/server/member/RaftMember.java | 77 +++-- .../cluster/server/service/BaseAsyncService.java | 19 +- .../cluster/server/service/BaseSyncService.java | 23 +- .../cluster/client/async/AsyncClientPoolTest.java | 37 ++- .../cluster/client/async/AsyncDataClientTest.java | 18 ++ .../client/async/AsyncDataHeartbeatClientTest.java | 18 ++ .../cluster/client/async/AsyncMetaClientTest.java | 18 ++ .../client/async/AsyncMetaHeartbeatClientTest.java | 18 ++ .../iotdb/cluster/log/LogDispatcherTest.java | 10 +- .../cluster/log/applier/DataLogApplierTest.java | 35 +++ .../cluster/log/snapshot/DataSnapshotTest.java | 8 + .../cluster/partition/SlotPartitionTableTest.java | 2 +- .../query/ClusterDataQueryExecutorTest.java | 3 - .../cluster/server/member/DataGroupMemberTest.java | 5 +- .../cluster/server/member/RaftMemberTest.java | 9 +- code-coverage/pom.xml | 2 +- compile-tools/pom.xml | 8 +- .../Administration-Management/Administration.md | 7 +- docs/UserGuide/Cluster/Cluster-Setup-Example.md | 47 +++ docs/UserGuide/Cluster/Cluster-Setup.md | 11 +- .../Data-Concept/Data-Model-and-Terminology.md | 4 +- .../DDL-Data-Definition-Language.md | 2 +- .../Administration-Management/Administration.md | 3 +- docs/zh/UserGuide/Cluster/Cluster-Setup-Example.md | 46 +++ docs/zh/UserGuide/Cluster/Cluster-Setup.md | 11 +- .../Data-Concept/Data-Model-and-Terminology.md | 4 +- .../DDL-Data-Definition-Language.md | 2 +- example/hadoop/pom.xml | 3 +- example/udf/pom.xml | 2 +- hadoop/pom.xml | 2 +- hive-connector/pom.xml | 5 +- .../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 9 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 14 +- pom.xml | 3 +- .../resources/conf/iotdb-engine.properties | 283 +++++++++-------- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 26 +- .../apache/iotdb/db/auth/entity/PrivilegeType.java | 1 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 39 ++- .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 28 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 46 ++- .../db/engine/cache/TimeSeriesMetadataCache.java | 2 +- .../compaction/CompactionMergeTaskPoolManager.java | 38 ++- .../db/engine/compaction/TsFileManagement.java | 58 +++- .../level/LevelCompactionTsFileManagement.java | 349 ++++++++++++--------- .../no/NoCompactionTsFileManagement.java | 132 ++++---- .../engine/compaction/utils/CompactionUtils.java | 25 +- .../iotdb/db/engine/memtable/AbstractMemTable.java | 11 + .../apache/iotdb/db/engine/memtable/IMemTable.java | 4 + .../db/engine/merge/manage/MergeResource.java | 2 +- .../iotdb/db/engine/merge/task/MergeFileTask.java | 14 +- .../db/engine/storagegroup/StorageGroupInfo.java | 6 +- .../engine/storagegroup/StorageGroupProcessor.java | 118 ++++--- .../db/engine/storagegroup/TsFileProcessor.java | 50 +-- .../engine/storagegroup/TsFileProcessorInfo.java | 6 +- .../db/engine/storagegroup/TsFileResource.java | 11 +- .../storagegroup/timeindex/DeviceTimeIndex.java | 18 -- .../storagegroup/timeindex/FileTimeIndex.java | 5 - .../engine/storagegroup/timeindex/ITimeIndex.java | 8 - .../iotdb/db/engine/upgrade/UpgradeTask.java | 40 +++ .../org/apache/iotdb/db/metadata/MManager.java | 6 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 15 +- .../org/apache/iotdb/db/qp/logical/Operator.java | 10 - .../org/apache/iotdb/db/qp/physical/BatchPlan.java | 53 ++++ .../db/qp/physical/crud/InsertMultiTabletPlan.java | 48 ++- .../physical/crud/InsertRowsOfOneDevicePlan.java | 42 ++- .../iotdb/db/qp/physical/crud/InsertRowsPlan.java | 49 ++- .../db/qp/physical/crud/InsertTabletPlan.java | 24 +- .../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 39 ++- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 71 ++++- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 9 + .../iotdb/db/query/executor/LastQueryExecutor.java | 41 ++- .../iotdb/db/query/executor/QueryRouter.java | 6 +- .../row/ElasticSerializableRowRecordList.java | 13 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 230 +++++++------- .../org/apache/iotdb/db/service/TSServiceImpl.java | 15 + .../apache/iotdb/db/tools/TsFileRewriteTool.java | 15 +- .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 16 + .../java/org/apache/iotdb/db/utils/AuthUtils.java | 2 - .../java/org/apache/iotdb/db/utils/MergeUtils.java | 4 +- .../org/apache/iotdb/db/utils/UpgradeUtils.java | 10 - .../apache/iotdb/db/auth/AuthorityCheckerTest.java | 23 +- .../auth/authorizer/LocalFileAuthorizerTest.java | 6 +- .../db/engine/compaction/CompactionChunkTest.java | 8 +- .../compaction/LevelCompactionCacheTest.java | 5 +- .../engine/compaction/LevelCompactionLogTest.java | 5 +- .../compaction/LevelCompactionMergeTest.java | 78 ++++- .../engine/compaction/LevelCompactionModsTest.java | 45 ++- .../compaction/LevelCompactionMoreDataTest.java | 5 +- .../LevelCompactionTsFileManagementTest.java | 69 ++++ .../NoCompactionTsFileManagementTest.java | 75 ++++- .../iotdb/db/engine/merge/MergeTaskTest.java | 75 +++++ .../engine/storagegroup/TsFileProcessorTest.java | 8 +- .../iotdb/db/integration/IOTDBGroupByIT.java | 19 ++ .../iotdb/db/integration/IoTDBGroupByFillIT.java | 22 ++ .../iotdb/db/integration/IoTDBGroupByMonthIT.java | 98 +++++- ...oTDBLoadExternalTsFileWithTimePartitionIT.java} | 44 ++- .../iotdb/db/integration/IoTDBSimpleQueryIT.java | 36 +++ .../integration/IoTDBUDTFAlignByTimeQueryIT.java | 16 + .../aggregation/IoTDBAggregationByLevelIT.java | 19 ++ .../db/integration/auth/IoTDBAuthorizationIT.java | 72 +++++ .../iotdb/db/metadata/MManagerBasicTest.java | 71 +++++ .../org/apache/iotdb/db/script/EnvScriptIT.java | 6 +- .../iotdb/db/utils/TsFileRewriteToolTest.java | 57 ++++ .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 20 +- .../org/apache/iotdb/rpc/RpcTransportFactory.java | 10 +- .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 4 +- .../rpc/TimeoutChangeableTFastFramedTransport.java | 8 +- .../TimeoutChangeableTSnappyFramedTransport.java | 20 +- site/src/main/.vuepress/config.js | 12 +- spark-tsfile/pom.xml | 2 +- thrift-cluster/src/main/thrift/cluster.thrift | 8 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 21 +- .../tsfile/read/filter/GroupByMonthFilter.java | 17 +- .../tsfile/v2/file/metadata/TsFileMetadataV2.java | 9 +- .../tsfile/v2/read/TsFileSequenceReaderForV2.java | 16 +- 140 files changed, 2797 insertions(+), 1128 deletions(-) diff --cc server/src/assembly/resources/conf/iotdb-engine.properties index 4ae5d52,123db71..4651b12 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@@ -249,26 -249,26 +249,26 @@@ timestamp_precision=m # Memory Allocation Ratio: Write, Read, Schema and Free Memory. # The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 1:1:1:1 , 6:2:1:1 --# If you have high level of writing pressure and low level of reading pressure, please adjust it to for example 6:1:1:2 - write_read_schema_free_memory_proportion=4:3:1:2 ++# If you have high level of writing pressure and low level of reading pressure, please adjust it to for example 6:1:1:2 + # write_read_schema_free_memory_proportion=4:3:1:2 # primitive array size (length of each array) in array pool - primitive_array_size=128 + # primitive_array_size=128 # Ratio of write memory for invoking flush disk, 0.4 by default # If you have extremely high write load (like batch=1000), it can be set lower than the default value like 0.2 - flush_proportion=0.4 + # flush_proportion=0.4 # Ratio of write memory allocated for buffered arrays, 0.6 by default - buffered_arrays_memory_proportion=0.6 + # buffered_arrays_memory_proportion=0.6 # Ratio of write memory for rejecting insertion, 0.8 by default --# If you have extremely high write load (like batch=1000) and the physical memory size is large enough, ++# If you have extremely high write load (like batch=1000) and the physical memory size is large enough, # it can be set higher than the default value like 0.9 - reject_proportion=0.8 + # reject_proportion=0.8 # If memory (in byte) of storage group increased more than this threshold, report to system. The default value is 16MB - storage_group_report_threshold=16777216 + # storage_group_report_threshold=16777216 # allowed max numbers of deduplicated path in one query # it's just an advised value, the real limitation will be the smaller one between this and the one we calculated @@@ -643,29 -654,21 +654,31 @@@ # index_root_dir=data/index # Is index enable - enable_index=false + # enable_index=false # How many threads can concurrently build index. When <= 0, use CPU core number. - concurrent_index_build_thread=0 + # concurrent_index_build_thread=0 # the default size of sliding window used for the subsequence matching in index framework - default_index_window_range=10 + # default_index_window_range=10 # buffer parameter for index processor. - index_buffer_size=134217728 + # index_buffer_size=134217728 # whether enable data partition. If disabled, all data belongs to partition 0 - enable_partition=false + # enable_partition=false # time range for partitioning data inside each storage group, the unit is second - partition_interval=604800 + # partition_interval=604800 + + # concurrent_writing_time_partition=1 + +#################### +### Double Write Configuration +#################### + +# enable_double_write=false +# secondary_address=127.0.0.1 +# secondary_port=6668 +# secondary_user=root - # secondary_password=root ++# secondary_password=root
