This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7bc44b5db6 [IOTDB-4753] Error serialized data size in TsFileData
(#7953)
7bc44b5db6 is described below
commit 7bc44b5db609ee20d07343c7d1e774b7e168de84
Author: yschengzi <[email protected]>
AuthorDate: Thu Nov 10 09:37:48 2022 +0800
[IOTDB-4753] Error serialized data size in TsFileData (#7953)
---
.../iotdb/db/engine/load/AlignedChunkData.java | 21 +++++++++++++++++++--
.../iotdb/db/engine/load/NonAlignedChunkData.java | 15 ++++++++++++++-
.../apache/iotdb/tsfile/file/header/PageHeader.java | 10 ++++++----
3 files changed, 39 insertions(+), 7 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java
b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java
index d81117e42b..0dc5b004f3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/load/AlignedChunkData.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.load;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.db.utils.TimePartitionUtils;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.exception.write.PageException;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
@@ -31,6 +32,7 @@ import
org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.PublicBAOS;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
@@ -85,6 +87,17 @@ public class AlignedChunkData implements ChunkData {
chunkHeaderList.add(chunkHeader);
pageNumbers.add(0);
+ addAttrDataSize();
+ }
+
+ private void addAttrDataSize() { // should be init before serialize,
corresponding serializeAttr
+ dataSize += 2 * Byte.BYTES; // isModification and isAligned
+ dataSize += Long.BYTES; // timePartitionSlot
+ int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length;
+ dataSize += ReadWriteForEncodingUtils.varIntSize(deviceLength);
+ dataSize += deviceLength; // device
+ dataSize += Integer.BYTES; // chunkHeaderListSize
+ dataSize += chunkHeaderList.get(0).getSerializedSize(); // timeChunkHeader
}
@Override
@@ -126,6 +139,10 @@ public class AlignedChunkData implements ChunkData {
public void addValueChunk(ChunkHeader chunkHeader) {
this.chunkHeaderList.add(chunkHeader);
this.pageNumbers.add(0);
+ dataSize += chunkHeader.getSerializedSize();
+ if (needDecodeChunk) {
+ dataSize += Integer.BYTES; // pageNumber
+ }
}
@Override
@@ -156,14 +173,14 @@ public class AlignedChunkData implements ChunkData {
public void writeEntireChunk(ByteBuffer chunkData, IChunkMetadata
chunkMetadata)
throws IOException {
dataSize += ReadWriteIOUtils.write(chunkData, stream);
- chunkMetadata.getStatistics().serialize(stream);
+ dataSize += chunkMetadata.getStatistics().serialize(stream);
}
@Override
public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData)
throws IOException {
pageNumbers.set(pageNumbers.size() - 1, pageNumbers.get(pageNumbers.size()
- 1) + 1);
dataSize += ReadWriteIOUtils.write(false, stream);
- pageHeader.serializeTo(stream);
+ dataSize += pageHeader.serializeTo(stream);
dataSize += ReadWriteIOUtils.write(pageData, stream);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java
b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java
index 3091988f9b..4c2e548b2d 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/load/NonAlignedChunkData.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.load;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.db.utils.TimePartitionUtils;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.exception.write.PageException;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
@@ -30,6 +31,7 @@ import
org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.PublicBAOS;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -66,6 +68,17 @@ public class NonAlignedChunkData implements ChunkData {
this.pageNumber = 0;
this.byteStream = new PublicBAOS();
this.stream = new DataOutputStream(byteStream);
+
+ addAttrDataSize();
+ }
+
+ private void addAttrDataSize() { // should be init before serialize,
corresponding serializeAttr
+ dataSize += 2 * Byte.BYTES; // isModification and isAligned
+ dataSize += Long.BYTES; // timePartitionSlot
+ int deviceLength = device.getBytes(TSFileConfig.STRING_CHARSET).length;
+ dataSize += ReadWriteForEncodingUtils.varIntSize(deviceLength);
+ dataSize += deviceLength; // device
+ dataSize += chunkHeader.getSerializedSize(); // timeChunkHeader
}
@Override
@@ -132,7 +145,7 @@ public class NonAlignedChunkData implements ChunkData {
public void writeEntirePage(PageHeader pageHeader, ByteBuffer pageData)
throws IOException {
pageNumber += 1;
dataSize += ReadWriteIOUtils.write(false, stream);
- pageHeader.serializeTo(stream);
+ dataSize += pageHeader.serializeTo(stream);
dataSize += ReadWriteIOUtils.write(pageData, stream);
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
index a7ccd47117..9ab5ea6f99 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java
@@ -117,10 +117,12 @@ public class PageHeader {
return statistics.getStartTime();
}
- public void serializeTo(OutputStream outputStream) throws IOException {
- ReadWriteForEncodingUtils.writeUnsignedVarInt(uncompressedSize,
outputStream);
- ReadWriteForEncodingUtils.writeUnsignedVarInt(compressedSize,
outputStream);
- statistics.serialize(outputStream);
+ public int serializeTo(OutputStream outputStream) throws IOException {
+ int length = 0;
+ length += ReadWriteForEncodingUtils.writeUnsignedVarInt(uncompressedSize,
outputStream);
+ length += ReadWriteForEncodingUtils.writeUnsignedVarInt(compressedSize,
outputStream);
+ length += statistics.serialize(outputStream);
+ return length;
}
@Override
