This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch add_base32_path in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3e9353efd14eedacc96619f172916202da54150f Author: HTHou <[email protected]> AuthorDate: Wed Dec 3 18:01:15 2025 +0800 add plainObjectPath and configuration --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +++ .../plan/planner/plan/node/write/ObjectNode.java | 10 +- .../plan/node/write/RelationalInsertRowsNode.java | 2 +- .../node/write/RelationalInsertTabletNode.java | 4 +- .../storageengine/dataregion/Base32ObjectPath.java | 6 +- .../db/storageengine/dataregion/IObjectPath.java | 15 ++- .../storageengine/dataregion/PlainObjectPath.java | 106 +++++++++++++++++++++ .../org/apache/iotdb/db/utils/ObjectTypeUtils.java | 10 +- 8 files changed, 146 insertions(+), 18 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 375285dfa2f..9ec3888198e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1188,8 +1188,11 @@ public class IoTDBConfig { private ConcurrentHashMap<String, EncryptParameter> tsFileDBToEncryptMap = new ConcurrentHashMap<>( Collections.singletonMap("root.__audit", new EncryptParameter("UNENCRYPTED", null))); + private long maxObjectSizeInByte = 4 * 1024 * 1024 * 1024L; + private boolean restrictObjectLimit = false; + IoTDBConfig() {} public int getMaxLogEntriesNumPerBatch() { @@ -4269,4 +4272,12 @@ public class IoTDBConfig { public void setMaxObjectSizeInByte(long maxObjectSizeInByte) { this.maxObjectSizeInByte = maxObjectSizeInByte; } + + public boolean getRestrictObjectLimit() { + return restrictObjectLimit; + } + + public void setRestrictObjectLimit(boolean restrictObjectLimit) { + this.restrictObjectLimit = restrictObjectLimit; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java index cef10d612fa..15c24bd3794 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java @@ -29,13 +29,13 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.IObjectPath.Deserializer; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; -import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; -import org.apache.iotdb.db.storageengine.dataregion.IObjectPath.Deserializer; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -134,7 +134,7 @@ public class ObjectNode extends SearchNode implements WALEntryValue { long searchIndex = stream.readLong(); boolean isEOF = stream.readByte() == 1; long offset = stream.readLong(); - IObjectPath filePath = Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(stream); + IObjectPath filePath = Deserializer.DESERIALIZER.deserializeFrom(stream); int contentLength = stream.readInt(); ObjectNode objectNode = new ObjectNode(isEOF, offset, contentLength, filePath); objectNode.setSearchIndex(searchIndex); @@ -145,7 +145,7 @@ public class ObjectNode extends SearchNode implements WALEntryValue { long searchIndex = buffer.getLong(); boolean isEOF = buffer.get() == 1; long offset = buffer.getLong(); - IObjectPath filePath = Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(buffer); + IObjectPath filePath = Deserializer.DESERIALIZER.deserializeFrom(buffer); Optional<File> objectFile = TierManager.getInstance().getAbsoluteObjectFilePath(filePath.toString()); int contentLength = buffer.getInt(); @@ -169,7 +169,7 @@ public class ObjectNode extends SearchNode implements WALEntryValue { public static ObjectNode deserialize(ByteBuffer byteBuffer) { boolean isEoF = ReadWriteIOUtils.readBool(byteBuffer); long offset = ReadWriteIOUtils.readLong(byteBuffer); - IObjectPath filePath = Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(byteBuffer); + IObjectPath filePath = Deserializer.DESERIALIZER.deserializeFrom(byteBuffer); int contentLength = ReadWriteIOUtils.readInt(byteBuffer); byte[] content = ReadWriteIOUtils.readBytes(byteBuffer, contentLength); return new ObjectNode(isEoF, offset, content, filePath); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java index 2d457cc9366..4186c097a5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java @@ -211,7 +211,7 @@ public class RelationalInsertRowsNode extends InsertRowsNode { long offset = buffer.getLong(); byte[] content = ReadWriteIOUtils.readBytes(buffer, buffer.remaining()); IObjectPath relativePath = - IObjectPath.Factory.DEFAULT_FACTORY.create( + IObjectPath.Factory.FACTORY.create( dataRegionReplicaSet.getRegionId().getId(), insertRowNode.getTime(), insertRowNode.getDeviceID(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index cc4cdcff01c..89c32c2f593 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -32,8 +32,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; -import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; @@ -469,7 +469,7 @@ public class RelationalInsertTabletNode extends InsertTabletNode { long offset = buffer.getLong(); byte[] content = ReadWriteIOUtils.readBytes(buffer, buffer.remaining()); IObjectPath relativePath = - IObjectPath.Factory.DEFAULT_FACTORY.create( + IObjectPath.Factory.FACTORY.create( entry.getKey().getRegionId().getId(), times[j], getDeviceID(j), measurements[column]); ObjectNode objectNode = new ObjectNode(isEoF, offset, content, relativePath); objectNode.setDataRegionReplicaSet(entry.getKey()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java index 728df25e007..273b1dc4007 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java @@ -34,8 +34,8 @@ import java.nio.file.Paths; public class Base32ObjectPath implements IObjectPath { - Path path; - int serializedSize = -1; + private final Path path; + private int serializedSize = -1; private static final Deserializer DESERIALIZER = new Deserializer() { @@ -52,7 +52,7 @@ public class Base32ObjectPath implements IObjectPath { private static final Factory FACTORY = Base32ObjectPath::new; - public Base32ObjectPath(String first, String... more) { + private Base32ObjectPath(String first, String... more) { path = Paths.get(first, more); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java index ee0613a1c85..76a1f21a306 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java @@ -19,6 +19,9 @@ package org.apache.iotdb.db.storageengine.dataregion; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; + import org.apache.tsfile.file.metadata.IDeviceID; import java.io.IOException; @@ -28,6 +31,8 @@ import java.nio.ByteBuffer; public interface IObjectPath { + IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); + int serialize(ByteBuffer byteBuffer); int serialize(OutputStream outputStream) throws IOException; @@ -38,7 +43,10 @@ public interface IObjectPath { IObjectPath create(int regionId, long time, IDeviceID iDeviceID, String measurement); - Factory DEFAULT_FACTORY = Base32ObjectPath.getFACTORY(); + Factory FACTORY = + CONFIG.getRestrictObjectLimit() + ? Base32ObjectPath.getFACTORY() + : PlainObjectPath.getFACTORY(); } interface Deserializer { @@ -47,6 +55,9 @@ public interface IObjectPath { IObjectPath deserializeFrom(InputStream inputStream) throws IOException; - Deserializer DEFAULT_DESERIALIZER = Base32ObjectPath.getDESERIALIZER(); + Deserializer DESERIALIZER = + CONFIG.getRestrictObjectLimit() + ? Base32ObjectPath.getDESERIALIZER() + : PlainObjectPath.getDESERIALIZER(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PlainObjectPath.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PlainObjectPath.java new file mode 100644 index 00000000000..35bc91a8b08 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PlainObjectPath.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion; + +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +public class PlainObjectPath implements IObjectPath { + + private final String filePath; + + private static final Deserializer DESERIALIZER = + new Deserializer() { + @Override + public IObjectPath deserializeFrom(ByteBuffer byteBuffer) { + return deserialize(byteBuffer); + } + + @Override + public IObjectPath deserializeFrom(InputStream inputStream) throws IOException { + return deserialize(inputStream); + } + }; + + private static final Factory FACTORY = PlainObjectPath::new; + + private PlainObjectPath(String filePath) { + this.filePath = filePath; + } + + public PlainObjectPath(int regionId, long time, IDeviceID iDeviceID, String measurement) { + String objectFileName = time + ".bin"; + Object[] segments = iDeviceID.getSegments(); + StringBuilder relativePathString = + new StringBuilder(String.valueOf(regionId)).append(File.separator); + for (Object segment : segments) { + relativePathString + .append(segment == null ? "null" : segment.toString().toLowerCase()) + .append(File.separator); + } + relativePathString.append(measurement).append(File.separator); + relativePathString.append(objectFileName); + this.filePath = relativePathString.toString(); + } + + @Override + public int serialize(ByteBuffer byteBuffer) { + return ReadWriteIOUtils.write(filePath, byteBuffer); + } + + @Override + public int serialize(OutputStream outputStream) throws IOException { + return ReadWriteIOUtils.write(filePath, outputStream); + } + + @Override + public int getSerializedSize() { + return ReadWriteIOUtils.sizeToWrite(filePath); + } + + public static PlainObjectPath deserialize(ByteBuffer byteBuffer) { + String filePath = ReadWriteIOUtils.readString(byteBuffer); + return new PlainObjectPath(filePath); + } + + public static PlainObjectPath deserialize(InputStream stream) throws IOException { + String filePath = ReadWriteIOUtils.readString(stream); + return new PlainObjectPath(filePath); + } + + @Override + public String toString() { + return filePath; + } + + public static Factory getFACTORY() { + return FACTORY; + } + + public static Deserializer getDESERIALIZER() { + return DESERIALIZER; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java index 2c2ce69ac19..9a0b5c57795 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java @@ -19,20 +19,18 @@ package org.apache.iotdb.db.utils; -import java.nio.ByteBuffer; import org.apache.iotdb.commons.exception.ObjectFileNotExist; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; -import org.apache.iotdb.db.storageengine.dataregion.IObjectPath.Deserializer; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; -import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.utils.Binary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.Optional; @@ -46,7 +44,8 @@ public class ObjectTypeUtils { public static File getObjectPathFromBinary(Binary binary) { byte[] bytes = binary.getValues(); ByteBuffer buffer = ByteBuffer.wrap(bytes, 8, bytes.length - 8); - String relativeObjectFilePath = Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(buffer).toString(); + String relativeObjectFilePath = + IObjectPath.Deserializer.DESERIALIZER.deserializeFrom(buffer).toString(); Optional<File> file = TIER_MANAGER.getAbsoluteObjectFilePath(relativeObjectFilePath); if (!file.isPresent()) { throw new ObjectFileNotExist(relativeObjectFilePath); @@ -58,7 +57,8 @@ public class ObjectTypeUtils { Binary binary, boolean needTempFile) { byte[] bytes = binary.getValues(); ByteBuffer buffer = ByteBuffer.wrap(bytes, 8, bytes.length - 8); - String relativeObjectFilePath = Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(buffer).toString(); + String relativeObjectFilePath = + IObjectPath.Deserializer.DESERIALIZER.deserializeFrom(buffer).toString(); return TIER_MANAGER.getAbsoluteObjectFilePath(relativeObjectFilePath, needTempFile); }
