This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch restrict_tvlist_rowCount
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/restrict_tvlist_rowCount by
this push:
new b89adef6330 init
b89adef6330 is described below
commit b89adef63302fb4cf1e2152e05fe2c8f10f50c4a
Author: HTHou <[email protected]>
AuthorDate: Wed Jul 10 15:56:03 2024 +0800
init
---
.../memtable/AlignedWritableMemChunk.java | 25 ++++----
.../dataregion/memtable/IWritableMemChunk.java | 20 +++---
.../dataregion/memtable/WritableMemChunk.java | 71 +++++++++++-----------
.../db/utils/datastructure/AlignedTVList.java | 4 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 4 +-
.../iotdb/db/utils/datastructure/TVList.java | 6 +-
6 files changed, 66 insertions(+), 64 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index b93474fdb92..ba78113bd03 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -88,22 +88,22 @@ public class AlignedWritableMemChunk implements
IWritableMemChunk {
}
@Override
- public void putLong(long t, long v) {
+ public boolean putLongWithFlushCheck(long t, long v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@Override
- public void putInt(long t, int v) {
+ public boolean putIntWithFlushCheck(long t, int v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@Override
- public void putFloat(long t, float v) {
+ public boolean putFloatWithFlushCheck(long t, float v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@Override
- public void putDouble(long t, double v) {
+ public boolean putDoubleWithFlushCheck(long t, double v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@@ -113,33 +113,33 @@ public class AlignedWritableMemChunk implements
IWritableMemChunk {
}
@Override
- public void putBoolean(long t, boolean v) {
+ public boolean putBooleanWithFlushCheck(long t, boolean v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@Override
public boolean putAlignedValueWithFlushCheck(long t, Object[] v) {
list.putAlignedValue(t, v);
- return list.reachMaxChunkSizeThreshold();
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end) {
+ public boolean putLongsWithFlushCheck(long[] t, long[] v, BitMap bitMap, int
start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@Override
- public void putInts(long[] t, int[] v, BitMap bitMap, int start, int end) {
+ public boolean putIntsWithFlushCheck(long[] t, int[] v, BitMap bitMap, int
start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@Override
- public void putFloats(long[] t, float[] v, BitMap bitMap, int start, int
end) {
+ public boolean putFloatsWithFlushCheck(long[] t, float[] v, BitMap bitMap,
int start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@Override
- public void putDoubles(long[] t, double[] v, BitMap bitMap, int start, int
end) {
+ public boolean putDoublesWithFlushCheck(long[] t, double[] v, BitMap bitMap,
int start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@@ -150,7 +150,8 @@ public class AlignedWritableMemChunk implements
IWritableMemChunk {
}
@Override
- public void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int
end) {
+ public boolean putBooleansWithFlushCheck(
+ long[] t, boolean[] v, BitMap bitMap, int start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
TSDataType.VECTOR);
}
@@ -158,7 +159,7 @@ public class AlignedWritableMemChunk implements
IWritableMemChunk {
public boolean putAlignedValuesWithFlushCheck(
long[] t, Object[] v, BitMap[] bitMaps, int start, int end) {
list.putAlignedValues(t, v, bitMaps, start, end);
- return list.reachMaxChunkSizeThreshold();
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
index eac06484035..2d49bc6ba19 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
@@ -31,31 +31,31 @@ import java.util.List;
public interface IWritableMemChunk extends WALEntryValue {
- void putLong(long t, long v);
+ boolean putLongWithFlushCheck(long t, long v);
- void putInt(long t, int v);
+ boolean putIntWithFlushCheck(long t, int v);
- void putFloat(long t, float v);
+ boolean putFloatWithFlushCheck(long t, float v);
- void putDouble(long t, double v);
+ boolean putDoubleWithFlushCheck(long t, double v);
boolean putBinaryWithFlushCheck(long t, Binary v);
- void putBoolean(long t, boolean v);
+ boolean putBooleanWithFlushCheck(long t, boolean v);
boolean putAlignedValueWithFlushCheck(long t, Object[] v);
- void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end);
+ boolean putLongsWithFlushCheck(long[] t, long[] v, BitMap bitMap, int start,
int end);
- void putInts(long[] t, int[] v, BitMap bitMap, int start, int end);
+ boolean putIntsWithFlushCheck(long[] t, int[] v, BitMap bitMap, int start,
int end);
- void putFloats(long[] t, float[] v, BitMap bitMap, int start, int end);
+ boolean putFloatsWithFlushCheck(long[] t, float[] v, BitMap bitMap, int
start, int end);
- void putDoubles(long[] t, double[] v, BitMap bitMap, int start, int end);
+ boolean putDoublesWithFlushCheck(long[] t, double[] v, BitMap bitMap, int
start, int end);
boolean putBinariesWithFlushCheck(long[] t, Binary[] v, BitMap bitMap, int
start, int end);
- void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int end);
+ boolean putBooleansWithFlushCheck(long[] t, boolean[] v, BitMap bitMap, int
start, int end);
boolean putAlignedValuesWithFlushCheck(
long[] t, Object[] v, BitMap[] bitMaps, int start, int end);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index 4537809f5af..3f05af15fb6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -61,30 +61,24 @@ public class WritableMemChunk implements IWritableMemChunk {
public boolean writeWithFlushCheck(long insertTime, Object objectValue) {
switch (schema.getType()) {
case BOOLEAN:
- putBoolean(insertTime, (boolean) objectValue);
- break;
+ return putBooleanWithFlushCheck(insertTime, (boolean) objectValue);
case INT32:
case DATE:
- putInt(insertTime, (int) objectValue);
- break;
+ return putIntWithFlushCheck(insertTime, (int) objectValue);
case INT64:
case TIMESTAMP:
- putLong(insertTime, (long) objectValue);
- break;
+ return putLongWithFlushCheck(insertTime, (long) objectValue);
case FLOAT:
- putFloat(insertTime, (float) objectValue);
- break;
+ return putFloatWithFlushCheck(insertTime, (float) objectValue);
case DOUBLE:
- putDouble(insertTime, (double) objectValue);
- break;
+ return putDoubleWithFlushCheck(insertTime, (double) objectValue);
case TEXT:
case BLOB:
case STRING:
return putBinaryWithFlushCheck(insertTime, (Binary) objectValue);
default:
- throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
schema.getType());
+ throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
schema.getType().name());
}
- return false;
}
@Override
@@ -99,35 +93,29 @@ public class WritableMemChunk implements IWritableMemChunk {
switch (dataType) {
case BOOLEAN:
boolean[] boolValues = (boolean[]) valueList;
- putBooleans(times, boolValues, bitMap, start, end);
- break;
+ return putBooleansWithFlushCheck(times, boolValues, bitMap, start,
end);
case INT32:
case DATE:
int[] intValues = (int[]) valueList;
- putInts(times, intValues, bitMap, start, end);
- break;
+ return putIntsWithFlushCheck(times, intValues, bitMap, start, end);
case INT64:
case TIMESTAMP:
long[] longValues = (long[]) valueList;
- putLongs(times, longValues, bitMap, start, end);
- break;
+ return putLongsWithFlushCheck(times, longValues, bitMap, start, end);
case FLOAT:
float[] floatValues = (float[]) valueList;
- putFloats(times, floatValues, bitMap, start, end);
- break;
+ return putFloatsWithFlushCheck(times, floatValues, bitMap, start, end);
case DOUBLE:
double[] doubleValues = (double[]) valueList;
- putDoubles(times, doubleValues, bitMap, start, end);
- break;
+ return putDoublesWithFlushCheck(times, doubleValues, bitMap, start,
end);
case TEXT:
case BLOB:
case STRING:
Binary[] binaryValues = (Binary[]) valueList;
return putBinariesWithFlushCheck(times, binaryValues, bitMap, start,
end);
default:
- throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + dataType);
+ throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE +
dataType.name());
}
- return false;
}
@Override
@@ -142,34 +130,39 @@ public class WritableMemChunk implements
IWritableMemChunk {
}
@Override
- public void putLong(long t, long v) {
+ public boolean putLongWithFlushCheck(long t, long v) {
list.putLong(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putInt(long t, int v) {
+ public boolean putIntWithFlushCheck(long t, int v) {
list.putInt(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putFloat(long t, float v) {
+ public boolean putFloatWithFlushCheck(long t, float v) {
list.putFloat(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putDouble(long t, double v) {
+ public boolean putDoubleWithFlushCheck(long t, double v) {
list.putDouble(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
public boolean putBinaryWithFlushCheck(long t, Binary v) {
list.putBinary(t, v);
- return list.reachMaxChunkSizeThreshold();
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putBoolean(long t, boolean v) {
+ public boolean putBooleanWithFlushCheck(long t, boolean v) {
list.putBoolean(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
@@ -178,35 +171,41 @@ public class WritableMemChunk implements
IWritableMemChunk {
}
@Override
- public void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end) {
+ public boolean putLongsWithFlushCheck(long[] t, long[] v, BitMap bitMap, int
start, int end) {
list.putLongs(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putInts(long[] t, int[] v, BitMap bitMap, int start, int end) {
+ public boolean putIntsWithFlushCheck(long[] t, int[] v, BitMap bitMap, int
start, int end) {
list.putInts(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putFloats(long[] t, float[] v, BitMap bitMap, int start, int
end) {
+ public boolean putFloatsWithFlushCheck(long[] t, float[] v, BitMap bitMap,
int start, int end) {
list.putFloats(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putDoubles(long[] t, double[] v, BitMap bitMap, int start, int
end) {
+ public boolean putDoublesWithFlushCheck(long[] t, double[] v, BitMap bitMap,
int start, int end) {
list.putDoubles(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
public boolean putBinariesWithFlushCheck(
long[] t, Binary[] v, BitMap bitMap, int start, int end) {
list.putBinaries(t, v, bitMap, start, end);
- return list.reachMaxChunkSizeThreshold();
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int
end) {
+ public boolean putBooleansWithFlushCheck(
+ long[] t, boolean[] v, BitMap bitMap, int start, int end) {
list.putBooleans(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 3aa1f204c14..940281bdcca 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -716,8 +716,8 @@ public abstract class AlignedTVList extends TVList {
}
@Override
- public boolean reachMaxChunkSizeThreshold() {
- return reachMaxChunkSizeFlag;
+ public boolean reachChunkSizeOrPointNumThreshold() {
+ return reachMaxChunkSizeFlag || rowCount >= MAX_SERIES_POINT_NUMBER;
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
index b11d7f14f54..bb8cd6e513c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
@@ -100,8 +100,8 @@ public abstract class BinaryTVList extends TVList {
}
@Override
- public boolean reachMaxChunkSizeThreshold() {
- return memoryBinaryChunkSize >= TARGET_CHUNK_SIZE;
+ public boolean reachChunkSizeOrPointNumThreshold() {
+ return memoryBinaryChunkSize >= TARGET_CHUNK_SIZE || rowCount >=
MAX_SERIES_POINT_NUMBER;
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 99a34b47084..374f51be440 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -50,6 +50,8 @@ public abstract class TVList implements WALEntryValue {
protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not
consistent";
protected static final long TARGET_CHUNK_SIZE =
IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
+ protected static final long MAX_SERIES_POINT_NUMBER =
+
IoTDBDescriptor.getInstance().getConfig().getAvgSeriesPointNumberThreshold();
// list of timestamp array, add 1 when expanded -> data point timestamp array
// index relation: arrayIndex -> elementIndex
protected List<long[]> timestamps;
@@ -153,8 +155,8 @@ public abstract class TVList implements WALEntryValue {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
- public boolean reachMaxChunkSizeThreshold() {
- return false;
+ public boolean reachChunkSizeOrPointNumThreshold() {
+ return rowCount >= MAX_SERIES_POINT_NUMBER;
}
public void putBoolean(long time, boolean value) {
