This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch split_text_chunk
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/split_text_chunk by this push:
new 5ee3c89c7d9 dev non_aligned
5ee3c89c7d9 is described below
commit 5ee3c89c7d95ef4ae4049e920282deb4cc94310a
Author: HTHou <[email protected]>
AuthorDate: Fri Sep 27 19:01:34 2024 +0800
dev non_aligned
---
.../dataregion/memtable/WritableMemChunk.java | 41 +++++++++++++---------
.../db/utils/datastructure/AlignedTVList.java | 5 ---
.../iotdb/db/utils/datastructure/TVList.java | 4 +--
3 files changed, 25 insertions(+), 25 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index ef23afd96f4..c562174a1ef 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -50,6 +50,8 @@ public class WritableMemChunk implements IWritableMemChunk {
private static final long TARGET_CHUNK_SIZE =
IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
+ private static final long MAX_SERIES_POINT_NUMBER =
+
IoTDBDescriptor.getInstance().getConfig().getAvgSeriesPointNumberThreshold();
private static final Logger LOGGER =
LoggerFactory.getLogger(WritableMemChunk.class);
private static final IoTDBConfig CONFIG =
IoTDBDescriptor.getInstance().getConfig();
@@ -340,6 +342,7 @@ public class WritableMemChunk implements IWritableMemChunk {
TSDataType tsDataType = schema.getType();
ChunkWriterImpl chunkWriterImpl = createIChunkWriter();
long binarySizePerChunk = 0;
+ int pointNumPerChunk = 0;
for (int sortedRowIndex = 0; sortedRowIndex < list.rowCount();
sortedRowIndex++) {
long time = list.getTime(sortedRowIndex);
@@ -377,29 +380,33 @@ public class WritableMemChunk implements
IWritableMemChunk {
Binary value = list.getBinary(sortedRowIndex);
chunkWriterImpl.write(time, value);
binarySizePerChunk += getBinarySize(value);
- if (binarySizePerChunk > TARGET_CHUNK_SIZE) {
- chunkWriterImpl.sealCurrentPage();
- chunkWriterImpl.clearPageWriter();
- try {
- ioTaskQueue.put(chunkWriterImpl);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- chunkWriterImpl = createIChunkWriter();
- binarySizePerChunk = 0;
- }
break;
default:
LOGGER.error("WritableMemChunk does not support data type: {}",
tsDataType);
break;
}
+ pointNumPerChunk++;
+ if (pointNumPerChunk > MAX_SERIES_POINT_NUMBER || binarySizePerChunk >
TARGET_CHUNK_SIZE) {
+ chunkWriterImpl.sealCurrentPage();
+ chunkWriterImpl.clearPageWriter();
+ try {
+ ioTaskQueue.put(chunkWriterImpl);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ chunkWriterImpl = createIChunkWriter();
+ binarySizePerChunk = 0;
+ pointNumPerChunk = 0;
+ }
}
- chunkWriterImpl.sealCurrentPage();
- chunkWriterImpl.clearPageWriter();
- try {
- ioTaskQueue.put(chunkWriterImpl);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ if (pointNumPerChunk != 0) {
+ chunkWriterImpl.sealCurrentPage();
+ chunkWriterImpl.clearPageWriter();
+ try {
+ ioTaskQueue.put(chunkWriterImpl);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 676831bd231..baba819d624 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -721,11 +721,6 @@ public abstract class AlignedTVList extends TVList {
}
}
- @Override
- public boolean reachChunkSizeOrPointNumThreshold() {
- return reachMaxChunkSizeFlag || rowCount >= MAX_SERIES_POINT_NUMBER;
- }
-
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
@Override
public void putAlignedValues(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index d75fc857c26..7a0780196c6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -51,8 +51,6 @@ public abstract class TVList implements WALEntryValue {
protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not
consistent";
protected static final long TARGET_CHUNK_SIZE =
IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
- protected static final long MAX_SERIES_POINT_NUMBER =
-
IoTDBDescriptor.getInstance().getConfig().getAvgSeriesPointNumberThreshold();
// list of timestamp array, add 1 when expanded -> data point timestamp array
// index relation: arrayIndex -> elementIndex
protected List<long[]> timestamps;
@@ -157,7 +155,7 @@ public abstract class TVList implements WALEntryValue {
}
public boolean reachChunkSizeOrPointNumThreshold() {
- return rowCount >= MAX_SERIES_POINT_NUMBER;
+ return false;
}
public void putBoolean(long time, boolean value) {
