This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2b9565dcb2 [IOTDB-4575] Remove max_chunk_raw_size_threshold by
target_chunk_size (#7530)
2b9565dcb2 is described below
commit 2b9565dcb2928e43807a9d9d83d8a6fc0cf5a7e3
Author: Mrquan <[email protected]>
AuthorDate: Fri Oct 7 14:52:47 2022 +0800
[IOTDB-4575] Remove max_chunk_raw_size_threshold by target_chunk_size
(#7530)
---
server/src/assembly/resources/conf/iotdb-datanode.properties | 8 ++------
.../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 -----------
.../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ------
.../apache/iotdb/db/utils/datastructure/AlignedTVList.java | 4 ++--
.../org/apache/iotdb/db/utils/datastructure/BinaryTVList.java | 2 +-
.../java/org/apache/iotdb/db/utils/datastructure/TVList.java | 4 ++--
6 files changed, 7 insertions(+), 28 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties
b/server/src/assembly/resources/conf/iotdb-datanode.properties
index 067e1ed1a3..65b0865f15 100644
--- a/server/src/assembly/resources/conf/iotdb-datanode.properties
+++ b/server/src/assembly/resources/conf/iotdb-datanode.properties
@@ -385,11 +385,6 @@ timestamp_precision=ms
# Datatype: int
# avg_series_point_number_threshold=100000
-# When a chunk in memtable reaches this threshold, flush the memtable to disk.
-# The default threshold is 20 MB.
-# Datatype: long
-# max_chunk_raw_size_threshold = 20971520
-
# How many threads can concurrently flush. When <= 0, use CPU core number.
# Datatype: int
# concurrent_flush_thread=0
@@ -599,7 +594,8 @@ timestamp_precision=ms
# Datatype: long, Unit: byte
# target_compaction_file_size=1073741824
-# The target chunk size in compaction, default is 1MB
+# The target chunk size in compaction and when memtable reaches this
threshold, flush the memtable to disk.
+# default is 1MB
# Datatype: long, Unit: byte
# target_chunk_size=1048576
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index e47c3adc1b..76b53f6db8 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -397,9 +397,6 @@ public class IoTDBConfig {
/** When average series point number reaches this, flush the memtable to
disk */
private int avgSeriesPointNumberThreshold = 100000;
- /** When a chunk in memtable reaches this threshold, flush the memtable to
disk */
- private long maxChunkRawSizeThreshold = 1024 * 1024 * 20L;
-
/** Enable inner space compaction for sequence files */
private boolean enableSeqSpaceCompaction = true;
@@ -2005,14 +2002,6 @@ public class IoTDBConfig {
this.avgSeriesPointNumberThreshold = avgSeriesPointNumberThreshold;
}
- public long getMaxChunkRawSizeThreshold() {
- return maxChunkRawSizeThreshold;
- }
-
- public void setMaxChunkRawSizeThreshold(long maxChunkRawSizeThreshold) {
- this.maxChunkRawSizeThreshold = maxChunkRawSizeThreshold;
- }
-
public long getCrossCompactionFileSelectionTimeBudget() {
return crossCompactionFileSelectionTimeBudget;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 8ee67e25cf..21889c5ae9 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -405,12 +405,6 @@ public class IoTDBDescriptor {
"avg_series_point_number_threshold",
Integer.toString(conf.getAvgSeriesPointNumberThreshold()))));
- conf.setMaxChunkRawSizeThreshold(
- Long.parseLong(
- properties.getProperty(
- "max_chunk_raw_size_threshold",
- Long.toString(conf.getMaxChunkRawSizeThreshold()))));
-
conf.setCheckPeriodWhenInsertBlocked(
Integer.parseInt(
properties.getProperty(
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index fff6a1cec6..b6ff3044e7 100644
---
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -186,7 +186,7 @@ public abstract class AlignedTVList extends TVList {
columnValue != null
? getBinarySize((Binary) columnValue)
: getBinarySize(Binary.EMPTY_VALUE);
- if (memoryBinaryChunkSize[i] >= maxChunkRawSizeThreshold) {
+ if (memoryBinaryChunkSize[i] >= targetChunkSize) {
reachMaxChunkSizeFlag = true;
}
break;
@@ -773,7 +773,7 @@ public abstract class AlignedTVList extends TVList {
memoryBinaryChunkSize[i] +=
arrayT[elementIndex + i1] != null ?
getBinarySize(arrayT[elementIndex + i1]) : 0;
}
- if (memoryBinaryChunkSize[i] > maxChunkRawSizeThreshold) {
+ if (memoryBinaryChunkSize[i] > targetChunkSize) {
reachMaxChunkSizeFlag = true;
}
break;
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
index 5b017e57d8..6c4e500495 100644
---
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
+++
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
@@ -99,7 +99,7 @@ public abstract class BinaryTVList extends TVList {
@Override
public boolean reachMaxChunkSizeThreshold() {
- return memoryBinaryChunkSize >= maxChunkRawSizeThreshold;
+ return memoryBinaryChunkSize >= targetChunkSize;
}
@Override
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 8dcea925a4..cb0f75c573 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -49,8 +49,8 @@ public abstract class TVList implements WALEntryValue {
protected static final int SMALL_ARRAY_LENGTH = 32;
protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not
consistent";
- protected static final long maxChunkRawSizeThreshold =
- IoTDBDescriptor.getInstance().getConfig().getMaxChunkRawSizeThreshold();
+ protected static final long targetChunkSize =
+ IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
// list of timestamp array, add 1 when expanded -> data point timestamp array
// index relation: arrayIndex -> elementIndex
protected List<long[]> timestamps;
