This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch config in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ed93441b469b292c9fad9844b9dc7bfe3ec8b952 Merge: 0e0ac5591d 3648dbea7a Author: HTHou <[email protected]> AuthorDate: Fri Oct 28 19:36:47 2022 +0800 fix .../confignode/conf/ConfigNodeDescriptor.java | 2 +- .../consensus/request/ConfigPhysicalPlan.java | 8 + .../consensus/request/ConfigPhysicalPlanType.java | 4 +- .../consensus/request/read/GetUDFJarPlan.java | 68 ++++ .../request/write/datanode/UpdateDataNodePlan.java | 71 ++++ .../consensus/response/DataNodeRegisterResp.java | 9 +- .../response/{TriggerJarResp.java => JarResp.java} | 10 +- .../iotdb/confignode/manager/ConfigManager.java | 42 +- .../apache/iotdb/confignode/manager/IManager.java | 17 +- .../iotdb/confignode/manager/TriggerManager.java | 12 +- .../iotdb/confignode/manager/UDFManager.java | 25 ++ .../iotdb/confignode/manager/node/NodeManager.java | 45 +++ .../iotdb/confignode/persistence/TriggerInfo.java | 8 +- .../iotdb/confignode/persistence/UDFInfo.java | 24 ++ .../persistence/executor/ConfigPlanExecutor.java | 6 + .../confignode/persistence/node/NodeInfo.java | 22 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 27 +- .../request/ConfigPhysicalPlanSerDeTest.java | 30 ++ docs/UserGuide/Process-Data/Select-Into.md | 425 ++++++++++++--------- docs/zh/UserGuide/Process-Data/Select-Into.md | 425 +++++++++++++-------- integration-test/checkstyle.xml | 2 +- integration-test/import-control.xml | 1 + .../apache/iotdb/it/env/AbstractNodeWrapper.java | 20 +- .../org/apache/iotdb/it/env/ConfigNodeWrapper.java | 20 + .../org/apache/iotdb/it/env/DataNodeWrapper.java | 42 +- .../confignode/it/IoTDBClusterPartitionIT.java | 19 +- .../iotdb/confignode/it/IoTDBClusterRestartIT.java | 148 ++++++- .../iotdb/confignode/it/IoTDBConfigNodeIT.java | 61 +-- .../confignode/it/IoTDBConfigNodeSnapshotIT.java | 18 +- .../confignode/it/utils/ConfigNodeTestUtils.java | 115 ++++++ .../dropwizard/DropwizardMetricManager.java | 2 +- .../resources/conf/iotdb-confignode-metric.yml | 2 +- .../resources/conf/iotdb-datanode-metric.yml | 2 +- .../iotdb/metrics/AbstractMetricManager.java | 47 +-- .../apache/iotdb/metrics/config/MetricConfig.java | 2 +- .../iotdb/metrics/impl/DoNothingMetricManager.java | 2 +- .../iotdb/metrics/utils/IoTDBMetricsUtils.java | 2 +- .../org/apache/iotdb/metrics/utils/MetricInfo.java | 2 +- .../micrometer/MicrometerMetricManager.java | 2 +- .../resources/conf/iotdb-common.properties | 2 +- .../apache/iotdb/commons/conf/IoTDBConstant.java | 2 - .../commons/udf/service/UDFClassLoaderManager.java | 5 +- .../resources/conf/iotdb-datanode.properties | 1 + .../apache/iotdb/db/client/ConfigNodeClient.java | 31 +- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 101 +++++ .../iotdb/db/engine/storagegroup/DataRegion.java | 4 + .../db/engine/storagegroup/TsFileProcessor.java | 8 +- .../iotdb/db/query/control/SessionManager.java | 1 + .../java/org/apache/iotdb/db/service/DataNode.java | 32 +- .../java/org/apache/iotdb/db/wal/WALManager.java | 24 +- ...oundRobinStrategy.java => ElasticStrategy.java} | 63 ++- .../db/wal/allocation/RoundRobinStrategy.java | 4 +- .../java/org/apache/iotdb/db/wal/node/WALNode.java | 9 +- .../apache/iotdb/db/metric/MetricServiceTest.java | 19 + .../org/apache/iotdb/db/wal/WALManagerTest.java | 14 +- ...nStrategyTest.java => ElasticStrategyTest.java} | 19 +- .../db/wal/allocation/FirstCreateStrategyTest.java | 3 - .../db/wal/allocation/RoundRobinStrategyTest.java | 3 - .../java/org/apache/iotdb/rpc/TSStatusCode.java | 3 +- .../src/main/thrift/confignode.thrift | 34 +- 60 files changed, 1543 insertions(+), 628 deletions(-) diff --cc node-commons/src/assembly/resources/conf/iotdb-common.properties index 17f6479227,b1f2f6c66c..3e6fc3cb83 --- a/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties @@@ -484,164 -665,25 +484,164 @@@ # Datatype: int # sub_compaction_thread_num=4 + #################### -### Metadata Cache Configuration +### Write Ahead Log Configuration #################### -# whether to cache meta data(BloomFilter, ChunkMetadata and TimeSeriesMetadata) or not. -# Datatype: boolean -# meta_data_cache_enable=true +# Write mode of wal +# The details of these three modes are as follows: +# 1. DISABLE: the system will disable wal. +# 2. SYNC: the system will submit wal synchronously, write request will not return until its wal is fsynced to the disk successfully. +# 3. ASYNC: the system will submit wal asynchronously, write request will return immediately no matter its wal is fsynced to the disk successfully. +# The write performance order is DISABLE > ASYNC > SYNC, but only SYNC mode can ensure data durability. +# wal_mode=ASYNC + +# Max number of wal nodes, each node corresponds to one wal directory - # The default value 0 means twice the number of wal dirs. ++# The default value 0 means the number is determined by the system. +# Notice: this value affects write performance significantly. +# For non-SSD disks, values between one third and half of storage groups number are recommended. +# Datatype: int +# max_wal_nodes_num=0 + +# Duration a wal flush operation will wait before calling fsync +# A duration greater than 0 batches multiple wal fsync calls into one. This is useful when disks are slow or WAL write contention exists. +# Notice: this value affects write performance significantly, values in the range of 0ms-10ms are recommended. +# Datatype: long +# fsync_wal_delay_in_ms=3 + +# Buffer size of each wal node +# If it's a value smaller than 0, use the default value 16 * 1024 * 1024 bytes (16MB). +# Datatype: int +# wal_buffer_size_in_byte=16777216 + +# Blocking queue capacity of each wal buffer, restricts maximum number of WALEdits cached in the blocking queue. +# Datatype: int +# wal_buffer_queue_capacity=50 + +# Size threshold of each wal file +# When a wal file's size exceeds this, the wal file will be closed and a new wal file will be created. +# If it's a value smaller than 0, use the default value 10 * 1024 * 1024 (10MB). +# Datatype: long +# wal_file_size_threshold_in_byte=10485760 + +# Minimum ratio of effective information in wal files +# This value should be between 0.0 and 1.0 +# If effective information ratio is below this value, MemTable snapshot or flush will be triggered. +# Increase this value when wal occupies too much disk space. But, if this parameter is too large, the write performance may decline. +# Datatype: double +# wal_min_effective_info_ratio=0.1 + +# MemTable size threshold for triggering MemTable snapshot in wal +# When a memTable's size (in byte) exceeds this, wal can flush this memtable to disk, otherwise wal will snapshot this memtable in wal. +# If it's a value smaller than 0, use the default value 8 * 1024 * 1024 bytes (8MB). +# Datatype: long +# wal_memtable_snapshot_threshold_in_byte=8388608 + +# MemTable's max snapshot number in wal +# If one memTable's snapshot number in wal exceeds this value, it will be flushed to disk. +# Datatype: int +# max_wal_memtable_snapshot_num=1 + +# The period when outdated wal files are periodically deleted +# If this value is too large, outdated wal files may not able to be deleted in time. +# If it's a value smaller than 0, use the default value 20 * 1000 ms (20 seconds). +# Datatype: long +# delete_wal_files_period_in_ms=20000 + +# The minimum size of wal files when throttle down in MultiLeader consensus +# If it's a value smaller than 0, use the default value 50 * 1024 * 1024 * 1024 bytes (50GB). +# Datatype: long +# multi_leader_throttle_threshold_in_byte=53687091200 + +# Maximum wait time of write cache in MultiLeader consensus +# If this value is less than or equal to 0, use the default value Long.MAX_VALUE. +# Datatype: long +# multi_leader_cache_window_time_in_ms=-1 -# Read memory Allocation Ratio: BloomFilterCache : ChunkCache : TimeSeriesMetadataCache : Coordinator : Operators : DataExchange : timeIndex in TsFileResourceList : others. -# The parameter form is a:b:c:d:e:f:g:h, where a, b, c, d, e, f, g and h are integers. for example: 1:1:1:1:1:1:1:1 , 1:100:200:50:200:200:200:50 -# chunk_timeseriesmeta_free_memory_proportion=1:100:200:50:200:200:200:50 #################### -### LAST Cache Configuration +### Timestamp Precision #################### -# Whether to enable LAST cache +# Use this value to set timestamp precision as "ms", "us" or "ns". +# Once the precision is been set, it can not be changed. +# Datatype: String +timestamp_precision=ms + +# Default TTL for storage groups that are not set TTL by statements, If not set (default), +# the TTL will be unlimited. +# Notice: if this property is changed, previous created storage group which are not set TTL will +# also be affected. And negative values are accepted, which means you can only insert future +# data. +# Datatype: long +# Unit: ms +# default_ttl=36000000 + +#################### +### Tlog Size Configuration +#################### +# max size for tag and attribute of one time series +# the unit is byte +# Datatype: int +# tag_attribute_total_size=700 + +#################### +### Out of Order Data Configuration +#################### + +# Add a switch to drop ouf-of-order data +# Out-of-order data will impact the aggregation query a lot. Users may not care about discarding some out-of-order data. # Datatype: boolean -# enable_last_cache=true +# enable_discard_out_of_order_data=false + +#################### +### Client Configuration +#################### + +# The maximum session idle time. unit: ms +# Idle sessions are the ones that performs neither query or non-query operations for a period of time +# Set to 0 to disable session timeout +# Datatype: int +# session_timeout_threshold=0 + +#################### +### Insert Control +#################### + +# When the waiting time (in ms) of an inserting exceeds this, throw an exception. 10000 by default. +# If the insertion has been rejected and the read load is low, it can be set larger +# Datatype: int +# max_waiting_time_when_insert_blocked=10000 + +#################### +### Query Configurations +#################### + +# allowed max numbers of deduplicated path in one query +# it's just an advised value, the real limitation will be the smaller one between this and the one we calculated +# Datatype: int +# max_deduplicated_path_num=1000 + +# the default time period that used in fill query, -1 by default means infinite past time +# Datatype: int, Unit: ms +# default_fill_interval=-1 + +# the max execution time of a DriverTask +# Datatype: int, Unit: ms +# driver_task_execution_time_slice_in_ms=100 + +# the max capacity of a TsBlock +# Datatype: int, Unit: byte +# max_tsblock_size_in_bytes=1048576 + +# the max number of lines in a single TsBlock +# Datatype: int +# max_tsblock_line_numbers=1000 + +# time cost(ms) threshold for slow query +# Datatype: long +# slow_query_threshold=5000 #################### ### External sort Configuration diff --cc server/src/assembly/resources/conf/iotdb-datanode.properties index c32893142b,b1f2f6c66c..4124493be5 --- a/server/src/assembly/resources/conf/iotdb-datanode.properties +++ b/server/src/assembly/resources/conf/iotdb-datanode.properties @@@ -63,6 -71,135 +63,7 @@@ schema_region_consensus_port=5001 # Notice: The ip for any target_config_node should never be 0.0.0.0 target_config_nodes=127.0.0.1:22277 -# Datatype: boolean -# rpc_thrift_compression_enable=false - -# if true, a snappy based compression method will be called before sending data by the network -# Datatype: boolean -# this feature is under development, set this as false before it is done. -# rpc_advanced_compression_enable=false - -# Datatype: int -# rpc_selector_thread_num=1 - -# Datatype: int -# rpc_min_concurrent_client_num=1 - -# Datatype: int -# rpc_max_concurrent_client_num=65535 - -# thrift max frame size, 512MB by default -# Datatype: int -# thrift_max_frame_size=536870912 - -# thrift init buffer size -# Datatype: int -# thrift_init_buffer_size=1024 - -# Thrift socket and connection timeout between raft nodes, in milliseconds. -# Datatype: int -# connection_timeout_ms=20000 - -# The maximum number of clients that can be idle for a node's InternalService. -# When the number of idle clients on a node exceeds this number, newly returned clients will be released -# Datatype: int -# core_connection_for_internal_service=100 - -# The maximum number of clients that can be applied for a node's InternalService -# Datatype: int -# max_connection_for_internal_service=100 - -# selector thread (TAsyncClientManager) nums for async thread in a clientManager -# Datatype: int -# selector_thread_nums_of_client_manager=1 - -#################### -### Write Ahead Log Configuration -#################### - -# Write mode of wal -# The details of these three modes are as follows: -# 1. DISABLE: the system will disable wal. -# 2. SYNC: the system will submit wal synchronously, write request will not return until its wal is fsynced to the disk successfully. -# 3. ASYNC: the system will submit wal asynchronously, write request will return immediately no matter its wal is fsynced to the disk successfully. -# The write performance order is DISABLE > ASYNC > SYNC, but only SYNC mode can ensure data durability. -# wal_mode=ASYNC - -# wal dirs -# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode). -# If it is absolute, system will save the data in the exact location it points to. -# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder. -# If there are more than one directory, please separate them by commas ",". -# Note: If wal_dirs is assigned an empty string(i.e.,zero-size), it will be handled as a relative path. -# For windows platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative. -# wal_dirs=data\\datanode\\wal -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# wal_dirs=data/datanode/wal - -# Max number of wal nodes, each node corresponds to one wal directory -# The default value 0 means the number is determined by the system. -# Notice: this value affects write performance significantly. -# For non-SSD disks, values between one third and half of storage groups number are recommended. -# Datatype: int -# max_wal_nodes_num=0 - -# Duration a wal flush operation will wait before calling fsync -# A duration greater than 0 batches multiple wal fsync calls into one. This is useful when disks are slow or WAL write contention exists. -# Notice: this value affects write performance significantly, values in the range of 0ms-10ms are recommended. -# Datatype: long -# fsync_wal_delay_in_ms=3 - -# Buffer size of each wal node -# If it's a value smaller than 0, use the default value 16 * 1024 * 1024 bytes (16MB). -# Datatype: int -# wal_buffer_size_in_byte=16777216 - -# Blocking queue capacity of each wal buffer, restricts maximum number of WALEdits cached in the blocking queue. -# Datatype: int -# wal_buffer_queue_capacity=50 - -# Size threshold of each wal file -# When a wal file's size exceeds this, the wal file will be closed and a new wal file will be created. -# If it's a value smaller than 0, use the default value 10 * 1024 * 1024 (10MB). -# Datatype: long -# wal_file_size_threshold_in_byte=10485760 - -# Minimum ratio of effective information in wal files -# This value should be between 0.0 and 1.0 -# If effective information ratio is below this value, MemTable snapshot or flush will be triggered. -# Increase this value when wal occupies too much disk space. But, if this parameter is too large, the write performance may decline. -# Datatype: double -# wal_min_effective_info_ratio=0.1 - -# MemTable size threshold for triggering MemTable snapshot in wal -# When a memTable's size (in byte) exceeds this, wal can flush this memtable to disk, otherwise wal will snapshot this memtable in wal. -# If it's a value smaller than 0, use the default value 8 * 1024 * 1024 bytes (8MB). -# Datatype: long -# wal_memtable_snapshot_threshold_in_byte=8388608 - -# MemTable's max snapshot number in wal -# If one memTable's snapshot number in wal exceeds this value, it will be flushed to disk. -# Datatype: int -# max_wal_memtable_snapshot_num=1 - -# The period when outdated wal files are periodically deleted -# If this value is too large, outdated wal files may not able to be deleted in time. -# If it's a value smaller than 0, use the default value 20 * 1000 ms (20 seconds). -# Datatype: long -# delete_wal_files_period_in_ms=20000 - -# The minimum size of wal files when throttle down in MultiLeader consensus -# If it's a value smaller than 0, use the default value 50 * 1024 * 1024 * 1024 bytes (50GB). -# Datatype: long -# multi_leader_throttle_threshold_in_byte=53687091200 - -# Maximum wait time of write cache in MultiLeader consensus -# If this value is less than or equal to 0, use the default value Long.MAX_VALUE. -# Datatype: long -# multi_leader_cache_window_time_in_ms=-1 + #################### ### Directory Configuration ####################
