This is an automated email from the ASF dual-hosted git repository.
chenyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 05b5b893df5 Optimize PBTree pst file space utilization for view
(#11896)
05b5b893df5 is described below
commit 05b5b893df50d03ca6472d1d9a4a305357916264
Author: Chen YZ <[email protected]>
AuthorDate: Tue Jan 16 12:01:18 2024 +0800
Optimize PBTree pst file space utilization for view (#11896)
* save
* done
* done
* fix ut
---
.../mtree/impl/mem/mnode/info/LogicalViewInfo.java | 2 +-
.../mtree/impl/pbtree/schemafile/RecordUtils.java | 45 +++++++++++++++-------
.../pbtree/schemafile/pagemgr/PageManager.java | 12 +-----
.../metadata/mtree/schemafile/SchemaFileTest.java | 2 +-
.../schema/view/viewExpression/ViewExpression.java | 18 +++++++++
5 files changed, 52 insertions(+), 27 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/LogicalViewInfo.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/LogicalViewInfo.java
index b6aa2b4c941..5ac3a5f185f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/LogicalViewInfo.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/LogicalViewInfo.java
@@ -146,7 +146,7 @@ public class LogicalViewInfo implements IMeasurementInfo {
*/
@Override
public int estimateSize() {
- return 8 + 8 + 1 + 32 + 64;
+ return 8 + 8 + 1 + 32 +
ViewExpression.getSerializeSize(schema.getExpression());
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/RecordUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/RecordUtils.java
index 438ef594ae7..b48eb06ce40 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/RecordUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/RecordUtils.java
@@ -53,8 +53,9 @@ public class RecordUtils {
private static final short INTERNAL_NODE_LENGTH =
(short) 1 + 2 + 8 + 4 + 1; // always fixed length record
private static final short MEASUREMENT_BASIC_LENGTH =
- (short) 1 + 2 + 8 + 8; // final length depends on its alias and props
- private static final short VIEW_BASIC_LENGTH = (short) 1 + 2 + 8 + 1;
+ (short) 1 + 2 + 8 + 8 + 4; // final length depends on its alias and props
+ private static final short VIEW_BASIC_LENGTH =
+ (short) 1 + 2 + 8 + 1; // final length depends on its view expression
/** These offset rather than magic number may also be used to track usage of
related field. */
private static final short LENGTH_OFFSET = 1;
@@ -86,6 +87,33 @@ public class RecordUtils {
}
}
+ public static int getRecordLength(ICachedMNode node) {
+ if (node.isMeasurement()) {
+ return getRecordLength(node.getAsMeasurementMNode());
+ } else {
+ return INTERNAL_NODE_LENGTH;
+ }
+ }
+
+ private static int getRecordLength(IMeasurementMNode<ICachedMNode> node) {
+ if (node.isLogicalView()) {
+ return VIEW_BASIC_LENGTH
+ + ViewExpression.getSerializeSize(((LogicalViewSchema)
node.getSchema()).getExpression());
+ } else {
+ // consider props and alias
+ int bufferLength =
+ node.getAlias() == null
+ ? 4 + MEASUREMENT_BASIC_LENGTH
+ : (node.getAlias().getBytes().length + 4 +
MEASUREMENT_BASIC_LENGTH);
+ if (node.getSchema().getProps() != null) {
+ for (Map.Entry<String, String> e :
node.getSchema().getProps().entrySet()) {
+ bufferLength += 8 + e.getKey().getBytes().length +
e.getValue().length();
+ }
+ }
+ return bufferLength;
+ }
+ }
+
/**
* Internal/Entity MNode Record Structure (in bytes): <br>
* (fixed length record)
@@ -154,18 +182,7 @@ public class RecordUtils {
* <p>It doesn't use MeasurementSchema.serializeTo for duplication of
measurementId
*/
private static ByteBuffer measurement2Buffer(IMeasurementMNode<ICachedMNode>
node) {
- int bufferLength =
- node.getAlias() == null
- ? 4 + MEASUREMENT_BASIC_LENGTH
- : (node.getAlias().getBytes().length + 4 +
MEASUREMENT_BASIC_LENGTH);
-
- // consider props
- bufferLength += 4;
- if (node.getSchema().getProps() != null) {
- for (Map.Entry<String, String> e :
node.getSchema().getProps().entrySet()) {
- bufferLength += 8 + e.getKey().getBytes().length +
e.getValue().length();
- }
- }
+ int bufferLength = getRecordLength(node);
// normal measurement
ByteBuffer buffer = ByteBuffer.allocate(bufferLength);
ReadWriteIOUtils.write(MEASUREMENT_TYPE, buffer);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageManager.java
index 97be15f4ef6..acd41d192af 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/pagemgr/PageManager.java
@@ -116,7 +116,6 @@ public abstract class PageManager implements IPageManager {
this.channel = channel;
this.pmtFile = pmtFile;
this.readChannel = FileChannel.open(pmtFile.toPath(),
StandardOpenOption.READ);
- this.metric = metric;
if (IoTDBDescriptor.getInstance()
.getConfig()
@@ -866,16 +865,7 @@ public abstract class PageManager implements IPageManager {
// for record offset, length of string key
int totalSize = SchemaFileConfig.SEG_HEADER_SIZE + 6 * childNum;
for (ICachedMNode child : node.getChildren().values()) {
- totalSize += child.getName().getBytes().length;
- if (child.isMeasurement()) {
- totalSize +=
- child.getAsMeasurementMNode().getAlias() == null
- ? 4
- : child.getAsMeasurementMNode().getAlias().getBytes().length
+ 4;
- totalSize += 24; // slightly larger than actually HEADER size
- } else {
- totalSize += 16; // slightly larger
- }
+ totalSize += child.getName().getBytes().length +
RecordUtils.getRecordLength(child);
}
return (short) totalSize > SchemaFileConfig.SEG_MIN_SIZ
? (short) totalSize
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/mtree/schemafile/SchemaFileTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/mtree/schemafile/SchemaFileTest.java
index 86490650270..c28029ee431 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/mtree/schemafile/SchemaFileTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/mtree/schemafile/SchemaFileTest.java
@@ -521,7 +521,7 @@ public class SchemaFileTest {
ICachedMNodeContainer.getCachedMNodeContainer(ent1).updateMNode("m1");
Assert.assertEquals(
- 64, getSegment(sf, getSegAddr(sf, getSegAddrInContainer(ent1),
"m1")).size());
+ 63, getSegment(sf, getSegAddr(sf, getSegAddrInContainer(ent1),
"m1")).size());
sf.writeMNode(ent1);
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/ViewExpression.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/ViewExpression.java
index 0c149023cb1..5efc0e19f50 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/ViewExpression.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/ViewExpression.java
@@ -47,6 +47,7 @@ import
org.apache.iotdb.commons.schema.view.viewExpression.unary.RegularViewExpr
import
org.apache.iotdb.commons.schema.view.viewExpression.visitor.ViewExpressionVisitor;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -63,6 +64,8 @@ import java.util.List;
*/
public abstract class ViewExpression {
+ protected int serSize = -1;
+
public <R, C> R accept(ViewExpressionVisitor<R, C> visitor, C context) {
return visitor.visitExpression(this, context);
}
@@ -119,6 +122,21 @@ public abstract class ViewExpression {
expression.serialize(stream);
}
+ public static int getSerializeSize(ViewExpression expression) {
+ if (expression.serSize == -1) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ ViewExpression.serialize(expression, baos);
+ expression.serSize = baos.toByteArray().length;
+ } catch (Exception e) {
+ // ByteArrayOutputStream is a memory-based output stream that does not
involve disk IO and
+ // will not throw an IOException except for OOM.
+ throw new RuntimeException(e);
+ }
+ }
+ return expression.serSize;
+ }
+
public static ViewExpression deserialize(ByteBuffer byteBuffer) {
short type = ReadWriteIOUtils.readShort(byteBuffer);
