This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch object_type in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9890fd67d9e0c4f3c18c3cab21940091d28355d1 Author: HTHou <[email protected]> AuthorDate: Thu Jul 3 18:44:19 2025 +0800 gen file name --- .../plan/planner/plan/node/write/FileNode.java | 10 ++++++++++ .../node/write/RelationalInsertTabletNode.java | 10 ++++++++++ .../plan/relational/planner/RelationPlanner.java | 12 ++--------- .../dataregion/memtable/TsFileProcessor.java | 20 +++++++++++++++---- .../org/apache/iotdb/db/utils/ObjectWriter.java | 23 ++++++++++++++-------- 5 files changed, 53 insertions(+), 22 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/FileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/FileNode.java index e57d0d7b222..cd82fbf12d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/FileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/FileNode.java @@ -38,6 +38,16 @@ public class FileNode { this.content = content; } + public FileNode(boolean isEOF, long offset, byte[] content) { + this.isEOF = isEOF; + this.offset = offset; + this.content = content; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + public String getFilePath() { return filePath; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index b04aebd0867..40690d012a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -37,6 +37,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -118,6 +119,15 @@ public class RelationalInsertTabletNode extends InsertTabletNode { return fileNodeList; } + public Binary[] getObjectColumn() { + for (int i = 0; i < columns.length; i++) { + if (dataTypes[i] == TSDataType.OBJECT) { + return (Binary[]) columns[i]; + } + } + return null; + } + @Override public IDeviceID getDeviceID(int rowIdx) { if (singleDevice) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 3274de4c9ac..1d6ff3a5c26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -131,7 +131,6 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.write.schema.MeasurementSchema; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -1141,16 +1140,9 @@ public class RelationPlanner extends AstVisitor<RelationPlan, Void> { long offset = BytesUtils.bytesToLong(offsetBytes); byte[] content = new byte[value.getLength() - 9]; System.arraycopy(value.getValues(), 9, content, 0, value.getLength() - 9); - // TODO:[OBJECT] Generate File name - String fileName = "/Users/ht/Documents/iotdb/data/object/file1"; - FileNode fileNode = new FileNode(fileName, isEoF, offset, content); + FileNode fileNode = new FileNode(isEoF, offset, content); fileNodeList.add(fileNode); - if (isEoF) { - ((Binary[]) insertTabletStatement.getColumns()[i])[j] = - new Binary(fileName.getBytes(StandardCharsets.UTF_8)); - } else { - ((Binary[]) insertTabletStatement.getColumns()[i])[j] = null; - } + ((Binary[]) insertTabletStatement.getColumns()[i])[j] = null; } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index eeddf861b10..e311e65df7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -100,6 +100,7 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.channels.ClosedChannelException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -562,11 +563,22 @@ public class TsFileProcessor { long[] infoForMetrics) throws WriteProcessException { if (insertTabletNode instanceof RelationalInsertTabletNode) { - List<FileNode> fileNodeList = - ((RelationalInsertTabletNode) insertTabletNode).getFileNodeList(); + RelationalInsertTabletNode relationalInsertTabletNode = + (RelationalInsertTabletNode) insertTabletNode; + List<FileNode> fileNodeList = relationalInsertTabletNode.getFileNodeList(); if (fileNodeList != null) { - for (FileNode fileNode : fileNodeList) { - try (ObjectWriter writer = new ObjectWriter(fileNode.getFilePath())) { + for (int i = 0; i < fileNodeList.size(); i++) { + FileNode fileNode = fileNodeList.get(i); + String objectFileName = + insertTabletNode.getTimes()[i] + "-" + config.getDataNodeId() + "-" + 1 + ".bin"; + File objectFile = new File(writer.getFile().getParent(), objectFileName); + if (fileNode.isEOF()) { + relationalInsertTabletNode.getObjectColumn()[i] = + new Binary( + (objectFile.getPath() + "," + objectFile.length()) + .getBytes(StandardCharsets.UTF_8)); + } + try (ObjectWriter writer = new ObjectWriter(objectFile)) { writer.write(fileNode.getContent()); } catch (Exception e) { throw new WriteProcessException(e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java index 7d572344ce1..070c7b5a149 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java @@ -19,25 +19,32 @@ package org.apache.iotdb.db.utils; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; public class ObjectWriter implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectWriter.class); + private final FileOutputStream fos; - public ObjectWriter(String filePath) throws FileNotFoundException { - // TODO:[OBJECT] Dir creation - Path path = Paths.get(filePath); - if (!Files.exists(path)) { + public ObjectWriter(File filePath) throws FileNotFoundException { + try { + FileUtils.forceMkdir(filePath.getParentFile()); + } catch (final IOException e) { + throw new FileNotFoundException("Error occurred during creating directory " + filePath); + } + if (!Files.exists(filePath.toPath())) { try { - Files.createFile(path); + Files.createFile(filePath.toPath()); } catch (IOException e) { - e.printStackTrace(); throw new FileNotFoundException(e.getMessage()); } }
