This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new ab887eb98ea Fix relational delete node serialization (#17867)
ab887eb98ea is described below
commit ab887eb98ea2d4f44a243d2df723a9bfa619df16
Author: Caideyipi <[email protected]>
AuthorDate: Tue Jun 9 12:23:31 2026 +0800
Fix relational delete node serialization (#17867)
* Fix relational delete node serialization
* Fix deletion entry serialized sizes
* Fix remaining serialized size calculations
* Fix ranged WAL serialized size calculations
* Fix relational delete resource identity
---
.../planner/plan/node/write/DeleteDataNode.java | 2 +-
.../plan/planner/plan/node/write/InsertNode.java | 10 +++++
.../planner/plan/node/write/InsertRowNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 40 ++++++++++++++---
.../plan/node/write/RelationalDeleteDataNode.java | 21 +++++++--
.../plan/node/write/RelationalInsertRowNode.java | 2 +-
.../node/write/RelationalInsertTabletNode.java | 7 ++-
.../dataregion/modification/DeletionPredicate.java | 9 ++--
.../dataregion/modification/IDPredicate.java | 13 ++----
.../dataregion/modification/ModEntry.java | 12 ++++-
.../dataregion/modification/TreeDeletionEntry.java | 11 +----
.../dataregion/modification/v1/Deletion.java | 10 ++++-
.../dataregion/wal/buffer/WALInfoEntry.java | 9 +++-
.../dataregion/wal/utils/WALWriteUtils.java | 16 +++++--
.../node/write/DeleteDataNodeSerdeTest.java | 28 ++++++++++++
.../planner/node/write/InsertRowNodeSerdeTest.java | 28 ++++++++----
.../node/write/InsertTabletNodeSerdeTest.java | 28 ++++++++----
.../node/write/RelationalDeleteDataNodeTest.java | 9 ++--
.../modification/TableDeletionEntryTest.java | 8 +++-
.../modification/TreeDeletionEntryTest.java | 5 ++-
.../dataregion/modification/v1/DeletionTest.java | 52 ++++++++++++++++++++++
.../dataregion/wal/io/WALFileTest.java | 1 +
22 files changed, 254 insertions(+), 69 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java
index 9193de12cf2..0462a81b243 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java
@@ -226,7 +226,7 @@ public class DeleteDataNode extends AbstractDeleteDataNode {
public int serializedSize() {
int size = FIXED_SERIALIZED_SIZE;
for (PartialPath path : pathList) {
- size += ReadWriteIOUtils.sizeToWrite(path.getFullPath());
+ size += WALWriteUtils.sizeToWrite(path.getFullPath());
}
return size;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
index e4f60752d2d..5d4d4ad8ffe 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
@@ -340,6 +340,16 @@ public abstract class InsertNode extends SearchNode {
return failedMeasurementNumber;
}
+ protected int getValidMeasurementNumber() {
+ int validMeasurementNumber = 0;
+ for (String measurement : measurements) {
+ if (measurement != null) {
+ validMeasurementNumber++;
+ }
+ }
+ return validMeasurementNumber;
+ }
+
public boolean isMeasurementFailed(int index) {
return measurements[index] == null;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java
index 6edb9efed84..fcb96cf3f07 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java
@@ -556,7 +556,7 @@ public class InsertRowNode extends InsertNode implements
WALEntryValue {
protected int subSerializeSize() {
int size = 0;
size += Long.BYTES;
- size += ReadWriteIOUtils.sizeToWrite(targetPath.getFullPath());
+ size += WALWriteUtils.sizeToWrite(targetPath.getFullPath());
return size + serializeMeasurementsAndValuesSize();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
index 677befc5d1c..98d0eca98b0 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
@@ -816,16 +816,30 @@ public class InsertTabletNode extends InsertNode
implements WALEntryValue {
return Short.BYTES + subSerializeSize(start, end);
}
+ /** Serialized size for wal */
+ public int serializedSize(List<int[]> rangeList) {
+ return Short.BYTES + subSerializeSize(rangeList);
+ }
+
int subSerializeSize(int start, int end) {
+ return subSerializeSizeByRange(Collections.singletonList(new int[] {start,
end}));
+ }
+
+ int subSerializeSize(List<int[]> rangeList) {
+ return subSerializeSizeByRange(rangeList);
+ }
+
+ private int subSerializeSizeByRange(List<int[]> rangeList) {
int size = 0;
size += Long.BYTES;
- size += ReadWriteIOUtils.sizeToWrite(targetPath.getFullPath());
+ size += WALWriteUtils.sizeToWrite(targetPath.getFullPath());
+ int rowNumInRange = getRowNumInRange(rangeList);
// measurements size
size += Integer.BYTES;
size += serializeMeasurementSchemasSize();
// times size
size += Integer.BYTES;
- size += Long.BYTES * (end - start);
+ size += Long.BYTES * rowNumInRange;
// bitmaps size
size += Byte.BYTES;
if (bitMaps != null) {
@@ -837,9 +851,13 @@ public class InsertTabletNode extends InsertNode
implements WALEntryValue {
size += Byte.BYTES;
if (bitMaps[i] != null) {
- int len = end - start;
- BitMap partBitMap = new BitMap(len);
- BitMap.copyOfRange(bitMaps[i], start, partBitMap, 0, len);
+ BitMap partBitMap = new BitMap(rowNumInRange);
+ int copiedLength = 0;
+ for (int[] range : rangeList) {
+ int len = range[1] - range[0];
+ BitMap.copyOfRange(bitMaps[i], range[0], partBitMap, copiedLength,
len);
+ copiedLength += len;
+ }
size += partBitMap.getByteArray().length;
}
}
@@ -847,7 +865,9 @@ public class InsertTabletNode extends InsertNode implements
WALEntryValue {
// values size
for (int i = 0; i < dataTypes.length; i++) {
if (columns[i] != null) {
- size += getColumnSize(dataTypes[i], columns[i], start, end);
+ for (int[] range : rangeList) {
+ size += getColumnSize(dataTypes[i], columns[i], range[0], range[1]);
+ }
}
}
// isAlign
@@ -857,6 +877,14 @@ public class InsertTabletNode extends InsertNode
implements WALEntryValue {
return size;
}
+ private int getRowNumInRange(List<int[]> rangeList) {
+ int rowNumInRange = 0;
+ for (int[] range : rangeList) {
+ rowNumInRange += range[1] - range[0];
+ }
+ return rowNumInRange;
+ }
+
private int getColumnSize(TSDataType dataType, Object column, int start, int
end) {
int size = 0;
switch (dataType) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNode.java
index f970ab70d28..a5a5a61cb9d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNode.java
@@ -36,6 +36,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
import
org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
import
org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -57,8 +58,8 @@ import java.util.stream.Collectors;
public class RelationalDeleteDataNode extends AbstractDeleteDataNode {
private static final Logger LOGGER =
LoggerFactory.getLogger(RelationalDeleteDataNode.class);
- /** byte: type */
- private static final int FIXED_SERIALIZED_SIZE = Short.BYTES;
+ /** short: type, long: searchIndex */
+ private static final int FIXED_SERIALIZED_SIZE = Short.BYTES + Long.BYTES;
private final List<TableDeletionEntry> modEntries;
@@ -220,9 +221,18 @@ public class RelationalDeleteDataNode extends
AbstractDeleteDataNode {
for (TableDeletionEntry modEntry : modEntries) {
size += modEntry.serializedSize();
}
+ size += sizeToWriteVarString(databaseName);
return size;
}
+ private static int sizeToWriteVarString(final String value) {
+ if (value == null) {
+ return ReadWriteForEncodingUtils.varIntSize(-1);
+ }
+ final int byteLength = value.getBytes(TSFileConfig.STRING_CHARSET).length;
+ return ReadWriteForEncodingUtils.varIntSize(byteLength) + byteLength;
+ }
+
@Override
public void serializeToWAL(IWALByteBufferView buffer) {
serializeToWAL(buffer, getEncodedSearchIndex());
@@ -236,6 +246,7 @@ public class RelationalDeleteDataNode extends
AbstractDeleteDataNode {
for (TableDeletionEntry modEntry : modEntries) {
modEntry.serialize(buffer);
}
+ ReadWriteIOUtils.writeVar(databaseName, buffer);
} catch (IOException e) {
LOGGER.error(DataNodeQueryMessages.FAILED_TO_SERIALIZE_MODENTRY_TO_WAL,
e);
}
@@ -283,12 +294,14 @@ public class RelationalDeleteDataNode extends
AbstractDeleteDataNode {
}
final RelationalDeleteDataNode that = (RelationalDeleteDataNode) obj;
return this.getPlanNodeId().equals(that.getPlanNodeId())
- && Objects.equals(this.modEntries, that.modEntries);
+ && Objects.equals(this.modEntries, that.modEntries)
+ && Objects.equals(this.databaseName, that.databaseName)
+ && Objects.equals(this.progressIndex, that.progressIndex);
}
@Override
public int hashCode() {
- return Objects.hash(getPlanNodeId(), modEntries, progressIndex);
+ return Objects.hash(getPlanNodeId(), modEntries, databaseName,
progressIndex);
}
public String toString() {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java
index 8ef6802f047..93838ffaf8b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java
@@ -219,7 +219,7 @@ public class RelationalInsertRowNode extends InsertRowNode {
@Override
protected int subSerializeSize() {
- return super.subSerializeSize() + columnCategories.length * Byte.BYTES;
+ return super.subSerializeSize() + getValidMeasurementNumber() * Byte.BYTES;
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
index ec20e7969da..839c3db5a7c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
@@ -329,7 +329,12 @@ public class RelationalInsertTabletNode extends
InsertTabletNode {
@Override
int subSerializeSize(int start, int end) {
- return super.subSerializeSize(start, end) + columnCategories.length *
Byte.BYTES;
+ return super.subSerializeSize(start, end) + getValidMeasurementNumber() *
Byte.BYTES;
+ }
+
+ @Override
+ int subSerializeSize(List<int[]> rangeList) {
+ return super.subSerializeSize(rangeList) + getValidMeasurementNumber() *
Byte.BYTES;
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java
index 877c94f7081..fcccaecc60f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java
@@ -147,16 +147,13 @@ public class DeletionPredicate implements
StreamSerializable, BufferSerializable
}
public int serializedSize() {
- // table name + id predicate +
+ // table name + id predicate + measurement names
int size =
- ReadWriteForEncodingUtils.varIntSize(tableName.length())
- + tableName.length() * Character.BYTES
+ ModEntry.sizeToWriteVarString(tableName)
+ idPredicate.serializedSize()
+ ReadWriteForEncodingUtils.varIntSize(measurementNames.size());
for (String measurementName : measurementNames) {
- size +=
- ReadWriteForEncodingUtils.varIntSize(
- measurementName.length() * measurementName.length() *
Character.BYTES);
+ size += ModEntry.sizeToWriteVarString(measurementName);
}
return size;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java
index d1f19df10a7..0d657dab9ed 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java
@@ -22,7 +22,6 @@ import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.utils.io.BufferSerializable;
import org.apache.iotdb.db.utils.io.StreamSerializable;
-import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.IDeviceID.Deserializer;
import org.apache.tsfile.utils.Accountable;
@@ -265,15 +264,9 @@ public abstract class IDPredicate implements
StreamSerializable, BufferSerializa
@Override
public int serializedSize() {
- if (pattern != null) {
- byte[] bytes = pattern.getBytes(TSFileConfig.STRING_CHARSET);
- return super.serializedSize()
- + ReadWriteForEncodingUtils.varIntSize(bytes.length)
- + bytes.length * Character.BYTES
- + ReadWriteForEncodingUtils.varIntSize(segmentIndex);
- } else {
- return ReadWriteForEncodingUtils.varIntSize(-1);
- }
+ return super.serializedSize()
+ + ModEntry.sizeToWriteVarString(pattern)
+ + ReadWriteForEncodingUtils.varIntSize(segmentIndex);
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
index c538410f359..5e9ba381c30 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
@@ -24,9 +24,11 @@ import org.apache.iotdb.db.utils.io.BufferSerializable;
import org.apache.iotdb.db.utils.io.StreamSerializable;
import org.apache.tsfile.annotations.TreeModel;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.utils.Accountable;
+import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.EOFException;
@@ -47,7 +49,15 @@ public abstract class ModEntry
public int serializedSize() {
// modType + time range
- return Byte.BYTES + Long.BYTES * 2 + Byte.BYTES * 2;
+ return Byte.BYTES + Long.BYTES * 2;
+ }
+
+ static int sizeToWriteVarString(final String value) {
+ if (value == null) {
+ return ReadWriteForEncodingUtils.varIntSize(-1);
+ }
+ final int byteLength = value.getBytes(TSFileConfig.STRING_CHARSET).length;
+ return ReadWriteForEncodingUtils.varIntSize(byteLength) + byteLength;
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
index 05436a6c9be..7e8059968a5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
@@ -34,7 +34,6 @@ import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.utils.RamUsageEstimator;
-import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,11 +84,7 @@ public class TreeDeletionEntry extends ModEntry {
@Override
public int serializedSize() {
- String patternFullPath = pathPattern.getFullPath();
- int length = patternFullPath.length();
- return super.serializedSize()
- + ReadWriteForEncodingUtils.varIntSize(length)
- + length * Character.BYTES;
+ return super.serializedSize() +
sizeToWriteVarString(pathPattern.getFullPath());
}
@Override
@@ -221,9 +216,7 @@ public class TreeDeletionEntry extends ModEntry {
}
public long getSerializedSize() {
- return modType.getSerializedSize()
- + Integer.BYTES
- + (long) pathPattern.getFullPath().length() * Character.BYTES;
+ return serializedSize();
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/Deletion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/Deletion.java
index 35cd5de5479..795395f4ee1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/Deletion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/Deletion.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.modification.v1;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -120,7 +121,14 @@ public class Deletion extends Modification implements
Cloneable {
}
public long getSerializedSize() {
- return Long.BYTES * 2 + Integer.BYTES + (long) getPathString().length() *
Character.BYTES;
+ return Long.BYTES * 2L + sizeToWriteString(getPathString());
+ }
+
+ private static int sizeToWriteString(String value) {
+ if (value == null) {
+ return Integer.BYTES;
+ }
+ return Integer.BYTES + value.getBytes(TSFileConfig.STRING_CHARSET).length;
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java
index 0fa12d84a01..957cf03d892 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java
@@ -82,7 +82,14 @@ public class WALInfoEntry extends WALEntry {
@Override
public int serializedSize() {
- return FIXED_SERIALIZED_SIZE + (value == null ? 0 :
value.serializedSize());
+ if (value == null) {
+ return FIXED_SERIALIZED_SIZE;
+ }
+ if (value instanceof InsertTabletNode && tabletInfo != null) {
+ return FIXED_SERIALIZED_SIZE
+ + ((InsertTabletNode)
value).serializedSize(tabletInfo.tabletRangeList);
+ }
+ return FIXED_SERIALIZED_SIZE + value.serializedSize();
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java
index be55b1785cd..087290c79ec 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java
@@ -21,6 +21,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.wal.utils;
import
org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.enums.CompressionType;
@@ -127,13 +128,20 @@ public class WALWriteUtils {
return write(NO_BYTE_TO_READ, buffer);
}
int len = 0;
- byte[] bytes = s.getBytes();
+ byte[] bytes = s.getBytes(TSFileConfig.STRING_CHARSET);
len += write(bytes.length, buffer);
buffer.put(bytes);
len += bytes.length;
return len;
}
+ public static int sizeToWrite(String s) {
+ if (s == null) {
+ return INT_LEN;
+ }
+ return INT_LEN + s.getBytes(TSFileConfig.STRING_CHARSET).length;
+ }
+
/**
* Write IDeviceID to byteBuffer.
*
@@ -197,15 +205,15 @@ public class WALWriteUtils {
public static int sizeToWrite(MeasurementSchema measurementSchema) {
int byteLen = 0;
- byteLen +=
ReadWriteIOUtils.sizeToWrite(measurementSchema.getMeasurementName());
+ byteLen += sizeToWrite(measurementSchema.getMeasurementName());
byteLen += 3 * Byte.BYTES;
Map<String, String> props = measurementSchema.getProps();
byteLen += Integer.BYTES;
if (props != null) {
for (Map.Entry<String, String> entry : props.entrySet()) {
- byteLen += ReadWriteIOUtils.sizeToWrite(entry.getKey());
- byteLen += ReadWriteIOUtils.sizeToWrite(entry.getValue());
+ byteLen += sizeToWrite(entry.getKey());
+ byteLen += sizeToWrite(entry.getValue());
}
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/DeleteDataNodeSerdeTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/DeleteDataNodeSerdeTest.java
index 16cb07f86c4..2d598936f99 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/DeleteDataNodeSerdeTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/DeleteDataNodeSerdeTest.java
@@ -26,10 +26,14 @@ import
org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
import
org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType;
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
+import
org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest;
import org.junit.Assert;
import org.junit.Test;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
@@ -66,4 +70,28 @@ public class DeleteDataNodeSerdeTest {
Assert.assertEquals(pathList.get(i), deserializedPathList.get(i));
}
}
+
+ @Test
+ public void testSerializeAndDeserializeForWAL() throws IllegalPathException,
IOException {
+ long startTime = 1;
+ long endTime = 10;
+ List<MeasurementPath> pathList = new ArrayList<>();
+ pathList.add(new
MeasurementPath("root.\u6570\u636e\u5e93.d1.\u6e29\u5ea6"));
+ pathList.add(new MeasurementPath("root.\u6570\u636e\u5e93.d2.*"));
+ DeleteDataNode deleteDataNode =
+ new DeleteDataNode(new PlanNodeId("DeleteDataNode"), pathList,
startTime, endTime);
+
+ ByteBuffer byteBuffer =
ByteBuffer.allocate(deleteDataNode.serializedSize());
+ deleteDataNode.serializeToWAL(new WALByteBufferForTest(byteBuffer));
+ Assert.assertEquals(deleteDataNode.serializedSize(),
byteBuffer.position());
+
+ DataInputStream dataInputStream =
+ new DataInputStream(new ByteArrayInputStream(byteBuffer.array()));
+ Assert.assertEquals(PlanNodeType.DELETE_DATA.getNodeType(),
dataInputStream.readShort());
+
+ DeleteDataNode deserializedNode =
DeleteDataNode.deserializeFromWAL(dataInputStream);
+ Assert.assertEquals(startTime, deserializedNode.getDeleteStartTime());
+ Assert.assertEquals(endTime, deserializedNode.getDeleteEndTime());
+ Assert.assertEquals(pathList, deserializedNode.getPathList());
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java
index 21bc9e0d120..f0681736329 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java
@@ -94,8 +94,8 @@ public class InsertRowNodeSerdeTest {
tmpNode.setPlanNodeId(insertRowNode.getPlanNodeId());
tmpNode.setMeasurementSchemas(
new MeasurementSchema[] {
- new MeasurementSchema("s1", TSDataType.DOUBLE),
- new MeasurementSchema("s2", TSDataType.FLOAT),
+ new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE),
+ new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT),
new MeasurementSchema("s3", TSDataType.INT64),
new MeasurementSchema("s4", TSDataType.INT32),
new MeasurementSchema("s5", TSDataType.BOOLEAN)
@@ -124,8 +124,8 @@ public class InsertRowNodeSerdeTest {
tmpNode.setPlanNodeId(insertRowNode.getPlanNodeId());
tmpNode.setMeasurementSchemas(
new MeasurementSchema[] {
- new MeasurementSchema("s1", TSDataType.DOUBLE),
- new MeasurementSchema("s2", TSDataType.FLOAT),
+ new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE),
+ new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT),
new MeasurementSchema("s3", TSDataType.INT64),
new MeasurementSchema("s4", TSDataType.INT32),
new MeasurementSchema("s5", TSDataType.BOOLEAN)
@@ -164,6 +164,18 @@ public class InsertRowNodeSerdeTest {
Assert.assertFalse(tmpNode.isLastFragment());
}
+ @Test
+ public void testRelationalSerializedSizeWithFailedMeasurement() {
+ RelationalInsertRowNode insertRowNode =
getRelationalInsertRowNodeWithMeasurementSchemas();
+ insertRowNode.markFailedMeasurement(1);
+ insertRowNode.setFailedMeasurementNumber(1);
+
+ ByteBuffer byteBuffer =
ByteBuffer.allocate(insertRowNode.serializedSize());
+ insertRowNode.serializeToWAL(new WALByteBufferForTest(byteBuffer));
+
+ Assert.assertEquals(insertRowNode.serializedSize(), byteBuffer.position());
+ }
+
private InsertRowNode getInsertRowNode() throws IllegalPathException {
long time = 110L;
TSDataType[] dataTypes =
@@ -214,9 +226,9 @@ public class InsertRowNodeSerdeTest {
InsertRowNode insertRowNode =
new InsertRowNode(
new PlanNodeId("plannode 2"),
- new PartialPath("root.isp.d2"),
+ new PartialPath("root.\u6570\u636e\u5e93.d2"),
false,
- new String[] {"s1", "s2", "s3", "s4", "s5"},
+ new String[] {"\u6e29\u5ea6", "\u6e7f\u5ea6", "s3", "s4", "s5"},
dataTypes,
time,
columns,
@@ -224,8 +236,8 @@ public class InsertRowNodeSerdeTest {
insertRowNode.setMeasurementSchemas(
new MeasurementSchema[] {
- new MeasurementSchema("s1", TSDataType.DOUBLE),
- new MeasurementSchema("s2", TSDataType.FLOAT),
+ new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE),
+ new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT),
new MeasurementSchema("s3", TSDataType.INT64),
new MeasurementSchema("s4", TSDataType.INT32),
new MeasurementSchema("s5", TSDataType.BOOLEAN)
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java
index ddc35e1eda9..8cf0ce6e36e 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java
@@ -87,8 +87,8 @@ public class InsertTabletNodeSerdeTest {
tmpNode.setMeasurementSchemas(
new MeasurementSchema[] {
- new MeasurementSchema("s1", TSDataType.DOUBLE),
- new MeasurementSchema("s2", TSDataType.FLOAT),
+ new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE),
+ new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT),
new MeasurementSchema("s3", TSDataType.INT64),
new MeasurementSchema("s4", TSDataType.INT32),
new MeasurementSchema("s5", TSDataType.BOOLEAN)
@@ -117,8 +117,8 @@ public class InsertTabletNodeSerdeTest {
tmpNode.setPlanNodeId(insertTabletNode.getPlanNodeId());
tmpNode.setMeasurementSchemas(
new MeasurementSchema[] {
- new MeasurementSchema("s1", TSDataType.DOUBLE),
- new MeasurementSchema("s2", TSDataType.FLOAT),
+ new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE),
+ new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT),
new MeasurementSchema("s3", TSDataType.INT64),
new MeasurementSchema("s4", TSDataType.INT32),
new MeasurementSchema("s5", TSDataType.BOOLEAN)
@@ -223,6 +223,18 @@ public class InsertTabletNodeSerdeTest {
Assert.assertFalse(tmpNode.isLastFragment());
}
+ @Test
+ public void testRelationalSerializedSizeWithFailedMeasurement() {
+ RelationalInsertTabletNode insertTabletNode =
getRelationalInsertTabletNodeWithSchema("table1");
+ insertTabletNode.markFailedMeasurement(1);
+ insertTabletNode.setFailedMeasurementNumber(1);
+
+ ByteBuffer byteBuffer =
ByteBuffer.allocate(insertTabletNode.serializedSize());
+ insertTabletNode.serializeToWAL(new WALByteBufferForTest(byteBuffer));
+
+ Assert.assertEquals(insertTabletNode.serializedSize(),
byteBuffer.position());
+ }
+
@Test
public void testInitTabletValuesWithAllTypes()
throws NoSuchMethodException, InvocationTargetException,
IllegalAccessException {
@@ -321,9 +333,9 @@ public class InsertTabletNodeSerdeTest {
InsertTabletNode insertTabletNode =
new InsertTabletNode(
new PlanNodeId("plannode 1"),
- new PartialPath("root.isp.d1"),
+ new PartialPath("root.\u6570\u636e\u5e93.d1"),
false,
- new String[] {"s1", "s2", "s3", "s4", "s5"},
+ new String[] {"\u6e29\u5ea6", "\u6e7f\u5ea6", "s3", "s4", "s5"},
dataTypes,
times,
null,
@@ -331,8 +343,8 @@ public class InsertTabletNodeSerdeTest {
times.length);
insertTabletNode.setMeasurementSchemas(
new MeasurementSchema[] {
- new MeasurementSchema("s1", TSDataType.DOUBLE),
- new MeasurementSchema("s2", TSDataType.FLOAT),
+ new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE),
+ new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT),
new MeasurementSchema("s3", TSDataType.INT64),
new MeasurementSchema("s4", TSDataType.INT32),
new MeasurementSchema("s5", TSDataType.BOOLEAN)
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java
index 78b7a400283..e97c11c1b47 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java
@@ -75,12 +75,9 @@ public class RelationalDeleteDataNodeTest {
new TableDeletionEntry(
new DeletionPredicate("table1", new NOP(),
Arrays.asList("s1", "s2")),
new TimeRange(0, 10))),
- null);
- relationalDeleteDataNode.setProgressIndex(new IoTProgressIndex(0, 1L));
+ "nezha");
- ByteBuffer buffer =
ByteBuffer.allocate(relationalDeleteDataNode.serializedSize());
- relationalDeleteDataNode.serialize(buffer);
- buffer.flip();
+ ByteBuffer buffer = relationalDeleteDataNode.serializeToByteBuffer();
assertEquals(relationalDeleteDataNode, PlanNodeType.deserialize(buffer));
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -92,6 +89,7 @@ public class RelationalDeleteDataNodeTest {
buffer = ByteBuffer.allocate(relationalDeleteDataNode.serializedSize());
WALByteBufferForTest walByteBufferForTest = new
WALByteBufferForTest(buffer);
relationalDeleteDataNode.serializeToWAL(walByteBufferForTest);
+ assertEquals(relationalDeleteDataNode.serializedSize(), buffer.position());
buffer.flip();
PlanNode deserialized = PlanNodeType.deserializeFromWAL(buffer);
// plan node id is not serialized to WAL, manually set it to pass
comparison
@@ -108,6 +106,7 @@ public class RelationalDeleteDataNodeTest {
deserialized.setPlanNodeId(relationalDeleteDataNode.getPlanNodeId());
assertEquals(relationalDeleteDataNode, deserialized);
+ relationalDeleteDataNode.setProgressIndex(new IoTProgressIndex(0, 1L));
buffer = relationalDeleteDataNode.serializeToDAL();
deserialized = DeleteNodeType.deserializeFromDAL(buffer);
// plan node id is not serialized to DAL, manually set it to pass
comparison
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java
index 5c2979a9075..a5d308a2b16 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java
@@ -32,6 +32,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Collections;
import static org.junit.Assert.assertEquals;
@@ -42,9 +43,13 @@ public class TableDeletionEntryTest {
@Test
public void testSerialization() throws IOException {
TableDeletionEntry entry =
- new TableDeletionEntry(new DeletionPredicate("table1", new NOP()), new
TimeRange(1, 5));
+ new TableDeletionEntry(
+ new DeletionPredicate(
+ "表格一", new SegmentExactMatch("区域一", 1), Arrays.asList("温度值",
"状态值")),
+ new TimeRange(1, 5));
ByteBuffer buffer = ByteBuffer.allocate(entry.serializedSize());
entry.serialize(buffer);
+ assertEquals(entry.serializedSize(), buffer.position());
buffer.flip();
ModEntry deserialized1 = ModEntry.createFrom(buffer);
assertEquals(entry, deserialized1);
@@ -52,6 +57,7 @@ public class TableDeletionEntryTest {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
entry.serialize(bos);
byte[] byteArray = bos.toByteArray();
+ assertEquals(entry.serializedSize(), byteArray.length);
ByteArrayInputStream bis = new ByteArrayInputStream(byteArray);
ModEntry deserialized2 = ModEntry.createFrom(bis);
assertEquals(entry, deserialized2);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntryTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntryTest.java
index 5087cce1111..961b4bbcdae 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntryTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntryTest.java
@@ -37,9 +37,11 @@ public class TreeDeletionEntryTest {
@Test
public void testSerialization() throws IllegalPathException, IOException {
- TreeDeletionEntry entry = new TreeDeletionEntry(new
MeasurementPath("root.db1.d1.s1"), 1, 5);
+ TreeDeletionEntry entry = new TreeDeletionEntry(new
MeasurementPath("root.数据库.d1.温度"), 1, 5);
+ assertEquals(entry.serializedSize(), entry.getSerializedSize());
ByteBuffer buffer = ByteBuffer.allocate(entry.serializedSize());
entry.serialize(buffer);
+ assertEquals(entry.serializedSize(), buffer.position());
buffer.flip();
ModEntry deserialized1 = ModEntry.createFrom(buffer);
assertEquals(entry, deserialized1);
@@ -47,6 +49,7 @@ public class TreeDeletionEntryTest {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
entry.serialize(bos);
byte[] byteArray = bos.toByteArray();
+ assertEquals(entry.serializedSize(), byteArray.length);
ByteArrayInputStream bis = new ByteArrayInputStream(byteArray);
ModEntry deserialized2 = ModEntry.createFrom(bis);
assertEquals(entry, deserialized2);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/DeletionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/DeletionTest.java
new file mode 100644
index 00000000000..2bc8dba9c22
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/v1/DeletionTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.modification.v1;
+
+import org.apache.iotdb.commons.path.MeasurementPath;
+
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+
+import static org.junit.Assert.assertEquals;
+
+public class DeletionTest {
+
+ @Test
+ public void testSerializedSize() throws Exception {
+ Deletion deletion =
+ new Deletion(new
MeasurementPath("root.\u6570\u636e\u5e93.d1.\u6e29\u5ea6"), 0, 1, 5);
+
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ long serializedSize =
+ deletion.serializeWithoutFileOffset(new
DataOutputStream(byteArrayOutputStream));
+ byte[] bytes = byteArrayOutputStream.toByteArray();
+
+ assertEquals(deletion.getSerializedSize(), serializedSize);
+ assertEquals(deletion.getSerializedSize(), bytes.length);
+ assertEquals(
+ deletion,
+ Deletion.deserializeWithoutFileOffset(
+ new DataInputStream(new ByteArrayInputStream(bytes))));
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java
index b9e0f3cdd48..16e9bb36f87 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java
@@ -178,6 +178,7 @@ public class WALFileTest {
ByteBuffer byteBuffer = ByteBuffer.allocate(walEntry.serializedSize());
WALByteBufferForTest buffer = new WALByteBufferForTest(byteBuffer);
walEntry.serialize(buffer);
+ assertEquals(walEntry.serializedSize(), byteBuffer.position());
byteBuffer.flip();
byte[] serializedEntry = new byte[byteBuffer.remaining()];
byteBuffer.get(serializedEntry);
