This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch object_delet in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b5a55494ac8be02197f5159c5af32bacb626130e Author: HTHou <[email protected]> AuthorDate: Mon Dec 22 17:50:36 2025 +0800 add some IT --- .../relational/it/session/IoTDBObjectDeleteIT.java | 273 +++++++++++++++++++++ .../db/storageengine/dataregion/DataRegion.java | 6 +- 2 files changed, 277 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java new file mode 100644 index 00000000000..8bfec578671 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.session; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import com.google.common.io.BaseEncoding; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertNull; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBObjectDeleteIT { + + @BeforeClass + public static void classSetUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @Before + public void setUp() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); + } + } + + @After + public void tearDown() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1"); + } + } + + @AfterClass + public static void classTearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void deleteObjectTest() + throws IoTDBConnectionException, StatementExecutionException, IOException { + String testObject = + System.getProperty("user.dir") + + File.separator + + "target" + + File.separator + + "test-classes" + + File.separator + + "object-example.pt"; + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // insert table data by tablet + List<String> columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file"); + List<TSDataType> dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.OBJECT); + List<ColumnCategory> columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue(rowIndex, 4, true, 0, Files.readAllBytes(Paths.get(testObject))); + session.insert(tablet); + tablet.reset(); + + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select READ_OBJECT(file) from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues()); + } + session.executeNonQueryStatement("DROP TABLE IF EXISTS object_table"); + } + } + + // test object file path + boolean success = false; + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String objectDirStr = dataNodeWrapper.getDataNodeObjectDir(); + File objectDir = new File(objectDirStr); + if (objectDir.exists() && objectDir.isDirectory()) { + File[] regionDirs = objectDir.listFiles(); + if (regionDirs != null) { + for (File regionDir : regionDirs) { + if (regionDir.isDirectory()) { + File objectFile = + new File( + regionDir, + convertPathString("object_table") + + File.separator + + convertPathString("1") + + File.separator + + convertPathString("5") + + File.separator + + convertPathString("3") + + File.separator + + convertPathString("file") + + File.separator + + "1.bin"); + if (objectFile.exists() && objectFile.isFile()) { + success = true; + } + } + } + } + } + } + Assert.assertFalse(success); + } + + @Test + public void deleteObjectSegmentsTest() + throws IoTDBConnectionException, StatementExecutionException, IOException { + String testObject = + System.getProperty("user.dir") + + File.separator + + "target" + + File.separator + + "test-classes" + + File.separator + + "object-example.pt"; + byte[] objectBytes = Files.readAllBytes(Paths.get(testObject)); + List<byte[]> objectSegments = new ArrayList<>(); + for (int i = 0; i < objectBytes.length; i += 512) { + objectSegments.add(Arrays.copyOfRange(objectBytes, i, Math.min(i + 512, objectBytes.length))); + } + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // insert table data by tablet + List<String> columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file"); + List<TSDataType> dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.OBJECT); + List<ColumnCategory> columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1); + for (int i = 0; i < objectSegments.size() - 1; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue(rowIndex, 4, false, i * 512L, objectSegments.get(i)); + session.insert(tablet); + tablet.reset(); + } + session.executeNonQueryStatement("DELETE FROM object_table where time = 1"); + + try (SessionDataSet dataSet = + session.executeQueryStatement("select file from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + assertNull(iterator.getString(1)); + } + } + } + + // test object file path + boolean success = false; + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String objectDirStr = dataNodeWrapper.getDataNodeObjectDir(); + File objectDir = new File(objectDirStr); + if (objectDir.exists() && objectDir.isDirectory()) { + File[] regionDirs = objectDir.listFiles(); + if (regionDirs != null) { + for (File regionDir : regionDirs) { + if (regionDir.isDirectory()) { + File objectTmpFile = + new File( + regionDir, + convertPathString("object_table") + + File.separator + + convertPathString("1") + + File.separator + + convertPathString("5") + + File.separator + + convertPathString("3") + + File.separator + + convertPathString("file") + + File.separator + + "1.bin.tmp"); + if (objectTmpFile.exists() && objectTmpFile.isFile()) { + success = true; + } + } + } + } + } + } + Assert.assertFalse(success); + } + + protected String convertPathString(String path) { + return BaseEncoding.base32().omitPadding().encode(path.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 8de74e066ab..e3ebd991cf0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2959,7 +2959,9 @@ public class DataRegion implements IDataRegionForQuery { matchedObjectDir.toPath(), Integer.MAX_VALUE, (path, attrs) -> - attrs.isRegularFile() && path.getFileName().toString().endsWith(".bin"))) { + attrs.isRegularFile() + && (path.getFileName().toString().endsWith(".bin") + || path.getFileName().toString().endsWith(".tmp")))) { paths.forEach( path -> { Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path); @@ -2985,7 +2987,7 @@ public class DataRegion implements IDataRegionForQuery { relativePath.getName(relativePath.getNameCount() - 2).toString()), StandardCharsets.UTF_8); String fileName = path.getFileName().toString(); - long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.'))); + long timestamp = Long.parseLong(fileName.substring(0, fileName.indexOf('.'))); logger.info( "timestamp {}, measurementId {}, ideviceId {}", timestamp,
