This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch force_ci/object_type in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3a8d6d5278874f8ff7a86998e84fdf89046ba026 Author: libo <[email protected]> AuthorDate: Mon Nov 10 16:03:44 2025 +0800 Delete the tsfile and related attachments When only one table and dat… (#16687) * Delete the tsfile and related attachments When only one table and data is cleared fully in the tsfile * Adjust logic is only effective in the table model, and refers to the tree model to delete files related to tsfile. * Supply an IT test to validate if involved supporting files about tsfile are deleted after delete all data in a table. * Resolve problem reported that file is not exist in the situation of 1C3D. * File may be not exist in the situation of 1C3D so that verify if it is exists first before scan the directory. * Fix the logic that verify only on table When multiple devices in same one table. * Don't delete file until all devices matched by time. * Avoid to delete the file when the device that need be deleted and other devices are all in the same one tsfile. * Increment a break to avoid extra I/O in the same file. * After delete all datas from a table, validate multiple devices involved files are deleted by different time range if it's success. * Add some logs and trace cause. * Debug * Change into logger * Decrement interval time * Clean environment before run testCompletelyDeleteTable * Recover all workflow yml files, log level need change into "debug" level. * Add a verification due to performance considerations. * Add some verification due to performance considerations. (cherry picked from commit f67526420d0f0fa5f01a2e8fa5929acd98173ab3) --- .../relational/it/db/it/IoTDBDeletionTableIT.java | 372 +++++++++++++++++++++ .../db/storageengine/dataregion/DataRegion.java | 121 ++++++- 2 files changed, 492 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 316ae614681..997a80a76f1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ManualIT; import org.apache.iotdb.itbase.category.TableClusterIT; @@ -34,6 +35,7 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.common.TimeRange; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -47,8 +49,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedWriter; +import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -63,8 +69,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -87,6 +95,17 @@ public class IoTDBDeletionTableIT { "INSERT INTO test.vehicle%d(time, deviceId, s0,s1,s2,s3,s4" + ") VALUES(%d,'d%d',%d,%d,%f,%s,%b)"; + private final String insertDeletionTemplate = + "INSERT INTO deletion.vehicle%d(time, deviceId, s0,s1,s2,s3,s4" + + ") VALUES(%d,'d%d',%d,%d,%f,%s,%b)"; + + private static String sequenceDataDir = "data" + File.separator + "sequence"; + private static String unsequenceDataDir = "data" + File.separator + "unsequence"; + + private static final String RESOURCE = ".resource"; + private static final String MODS = ".mods"; + private static final String TSFILE = ".tsfile"; + @BeforeClass public static void setUpClass() { Locale.setDefault(Locale.ENGLISH); @@ -470,6 +489,26 @@ public class IoTDBDeletionTableIT { } } + @Test + public void testFullDeleteWithoutWhereClauseByDifferentTime() throws SQLException { + prepareMultiDeviceDifferentTimeData(5, 2); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use deletion"); + statement.execute("DELETE FROM vehicle5"); + try (ResultSet set = statement.executeQuery("SELECT s0 FROM vehicle5")) { + int cnt = 0; + while (set.next()) { + cnt++; + } + assertEquals(0, cnt); + } + cleanData(5); + } catch (Exception e) { + fail(e.getMessage()); + } + } + @Test public void testDeleteWithSpecificDevice() throws SQLException { prepareData(6, 1); @@ -2003,6 +2042,226 @@ public class IoTDBDeletionTableIT { } } + @Test + public void testCompletelyDeleteTable() throws SQLException { + int testNum = 1; + cleanDeletionDatabase(); + prepareDeletionDatabase(); + prepareMultiDeviceDifferentTimeData(testNum, 1); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use deletion"); + + statement.execute("DROP TABLE vehicle" + testNum); + + statement.execute("flush"); + + statement.execute( + String.format( + "CREATE TABLE vehicle%d(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + testNum)); + + try (ResultSet set = statement.executeQuery("SELECT * FROM vehicle" + testNum)) { + assertFalse(set.next()); + } + + prepareData(testNum, 1); + + statement.execute("DELETE FROM vehicle" + testNum + " WHERE time <= 1000"); + + Awaitility.await() + .atMost(5, TimeUnit.MINUTES) + .pollDelay(500, TimeUnit.MILLISECONDS) + .pollInterval(500, TimeUnit.MILLISECONDS) + .until( + () -> { + AtomicBoolean completelyDeleteSuccess = new AtomicBoolean(true); + boolean allPass = true; + for (DataNodeWrapper wrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String dataNodeDir = wrapper.getDataNodeDir(); + + if (Paths.get( + dataNodeDir + + File.separator + + sequenceDataDir + + File.separator + + "deletion") + .toFile() + .exists()) { + try (Stream<Path> s = + Files.walk( + Paths.get( + dataNodeDir + + File.separator + + sequenceDataDir + + File.separator + + "deletion"))) { + s.forEach( + source -> { + if (source.toString().endsWith(RESOURCE) + || source.toString().endsWith(MODS) + || source.toString().endsWith(TSFILE)) { + if (source.toFile().length() > 0) { + LOGGER.error( + "[testCompletelyDeleteTable] undeleted seq file : {}", + source.toFile().getAbsolutePath()); + completelyDeleteSuccess.set(false); + } + } + }); + } + } + + if (Paths.get( + dataNodeDir + + File.separator + + unsequenceDataDir + + File.separator + + "deletion") + .toFile() + .exists()) { + try (Stream<Path> s = + Files.walk( + Paths.get( + dataNodeDir + + File.separator + + unsequenceDataDir + + File.separator + + "deletion"))) { + s.forEach( + source -> { + if (source.toString().endsWith(RESOURCE) + || source.toString().endsWith(MODS) + || source.toString().endsWith(TSFILE)) { + if (source.toFile().length() > 0) { + LOGGER.error( + "[testCompletelyDeleteTable] undeleted unseq file: {}", + source.toFile().getAbsolutePath()); + completelyDeleteSuccess.set(false); + } + } + }); + } + } + + allPass = allPass && completelyDeleteSuccess.get(); + } + return allPass; + }); + } + cleanData(testNum); + } + + @Test + public void testMultiDeviceCompletelyDeleteTable() throws SQLException { + int testNum = 1; + cleanDeletionDatabase(); + prepareDeletionDatabase(); + prepareMultiDeviceDifferentTimeData(testNum, 2); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use deletion"); + + statement.execute("DROP TABLE vehicle" + testNum); + + statement.execute("flush"); + + statement.execute( + String.format( + "CREATE TABLE vehicle%d(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + testNum)); + + try (ResultSet set = statement.executeQuery("SELECT * FROM vehicle" + testNum)) { + assertFalse(set.next()); + } + + prepareData(testNum, 2); + + statement.execute("DELETE FROM vehicle" + testNum + " WHERE time <= 1000"); + + Awaitility.await() + .atMost(5, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until( + () -> { + AtomicBoolean completelyDeleteSuccess = new AtomicBoolean(true); + boolean allPass = true; + for (DataNodeWrapper wrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String dataNodeDir = wrapper.getDataNodeDir(); + + if (Paths.get( + dataNodeDir + + File.separator + + sequenceDataDir + + File.separator + + "deletion") + .toFile() + .exists()) { + try (Stream<Path> s = + Files.walk( + Paths.get( + dataNodeDir + + File.separator + + sequenceDataDir + + File.separator + + "deletion"))) { + s.forEach( + source -> { + if (source.toString().endsWith(RESOURCE) + || source.toString().endsWith(MODS) + || source.toString().endsWith(TSFILE)) { + if (source.toFile().length() > 0) { + LOGGER.error( + "[testMultiDeviceCompletelyDeleteTable] undeleted unseq file: {}", + source.toFile().getAbsolutePath()); + completelyDeleteSuccess.set(false); + } + } + }); + } + } + + if (Paths.get( + dataNodeDir + + File.separator + + unsequenceDataDir + + File.separator + + "deletion") + .toFile() + .exists()) { + try (Stream<Path> s = + Files.walk( + Paths.get( + dataNodeDir + + File.separator + + unsequenceDataDir + + File.separator + + "deletion"))) { + s.forEach( + source -> { + if (source.toString().endsWith(RESOURCE) + || source.toString().endsWith(MODS) + || source.toString().endsWith(TSFILE)) { + if (source.toFile().length() > 0) { + LOGGER.error( + "[testMultiDeviceCompletelyDeleteTable] undeleted unseq file: {}", + source.toFile().getAbsolutePath()); + completelyDeleteSuccess.set(false); + } + } + }); + } + } + + allPass = allPass && completelyDeleteSuccess.get(); + } + return allPass; + }); + } + cleanData(testNum); + } + private static void prepareDatabase() { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { @@ -2015,6 +2274,40 @@ public class IoTDBDeletionTableIT { } } + private static void prepareDeletionDatabase() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE IF NOT EXISTS deletion"); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + private void cleanDeletionDatabase() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE IF EXISTS deletion"); + for (DataNodeWrapper wrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String dataNodeDir = wrapper.getDataNodeDir(); + File targetFile = + Paths.get(dataNodeDir + File.separator + sequenceDataDir + File.separator + "deletion") + .toFile(); + if (targetFile.exists()) { + targetFile.delete(); + } + + targetFile = + Paths.get(dataNodeDir + File.separator + sequenceDataDir + File.separator + "deletion") + .toFile(); + if (targetFile.exists()) { + targetFile.delete(); + } + } + } catch (Exception e) { + fail(e.getMessage()); + } + } + private void prepareData(int testNum, int deviceNum) throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { @@ -2062,6 +2355,85 @@ public class IoTDBDeletionTableIT { } } + private void prepareMultiDeviceDifferentTimeData(int testNum, int deviceNum) throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use deletion"); + statement.execute( + String.format( + "CREATE TABLE IF NOT EXISTS vehicle%d(deviceId STRING TAG, s0 INT32 FIELD, s1 INT64 FIELD, s2 FLOAT FIELD, s3 TEXT FIELD, s4 BOOLEAN FIELD)", + testNum)); + + for (int d = 0; d < deviceNum; d++) { + // prepare seq file + for (int i = 201 * (d + 1); i <= 300 * (d + 1); i++) { + statement.execute( + String.format( + insertDeletionTemplate, + testNum, + i, + d, + i, + i, + (double) i, + "'" + i + "'", + i % 2 == 0)); + } + } + + statement.execute("flush"); + + for (int d = 0; d < deviceNum; d++) { + // prepare unseq File + for (int i = 1 * (d + 1); i <= 100 * (d + 1); i++) { + statement.execute( + String.format( + insertDeletionTemplate, + testNum, + i, + d, + i, + i, + (double) i, + "'" + i + "'", + i % 2 == 0)); + } + } + statement.execute("flush"); + + for (int d = 0; d < deviceNum; d++) { + // prepare BufferWrite cache + for (int i = 301 * (d + 1); i <= 400 * (d + 1); i++) { + statement.execute( + String.format( + insertDeletionTemplate, + testNum, + i, + d, + i, + i, + (double) i, + "'" + i + "'", + i % 2 == 0)); + } + // prepare Overflow cache + for (int i = 101 * (d + 1); i <= 200 * (d + 1); i++) { + statement.execute( + String.format( + insertDeletionTemplate, + testNum, + i, + d, + i, + i, + (double) i, + "'" + i + "'", + i % 2 == 0)); + } + } + } + } + private void cleanData(int testNum) throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index eec169aa807..2c0c39032c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2941,12 +2941,110 @@ public class DataRegion implements IDataRegionForQuery { private void deleteDataInSealedFiles(Collection<TsFileResource> sealedTsFiles, ModEntry deletion) throws IOException { Set<ModificationFile> involvedModificationFiles = new HashSet<>(); + List<TsFileResource> deletedByMods = new ArrayList<>(); + List<TsFileResource> deletedByFiles = new ArrayList<>(); for (TsFileResource sealedTsFile : sealedTsFiles) { if (canSkipDelete(sealedTsFile, deletion)) { continue; } - involvedModificationFiles.add(sealedTsFile.getModFileForWrite()); + ITimeIndex timeIndex = sealedTsFile.getTimeIndex(); + + if ((timeIndex instanceof ArrayDeviceTimeIndex) + && (deletion.getType() == ModEntry.ModType.TABLE_DELETION)) { + ArrayDeviceTimeIndex deviceTimeIndex = (ArrayDeviceTimeIndex) timeIndex; + Set<IDeviceID> devicesInFile = deviceTimeIndex.getDevices(); + boolean onlyOneTable = false; + + if (deletion instanceof TableDeletionEntry) { + TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; + String tableName = tableDeletionEntry.getTableName(); + long matchSize = + devicesInFile.stream() + .filter( + device -> { + if (logger.isDebugEnabled()) { + logger.debug( + "device is {}, deviceTable is {}, tableDeletionEntry.getPredicate().matches(device) is {}", + device, + device.getTableName(), + tableDeletionEntry.getPredicate().matches(device)); + } + return tableName.equals(device.getTableName()) + && tableDeletionEntry.getPredicate().matches(device); + }) + .count(); + onlyOneTable = matchSize == devicesInFile.size(); + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, matchSize is {}, onlyOneTable is {}", + tableName, + matchSize, + onlyOneTable); + } + } + + if (onlyOneTable) { + int matchSize = 0; + for (IDeviceID device : devicesInFile) { + Optional<Long> optStart = deviceTimeIndex.getStartTime(device); + Optional<Long> optEnd = deviceTimeIndex.getEndTime(device); + if (!optStart.isPresent() || !optEnd.isPresent()) { + continue; + } + + long fileStartTime = optStart.get(); + long fileEndTime = optEnd.get(); + + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, device is {}, deletionStartTime is {}, deletionEndTime is {}, fileStartTime is {}, fileEndTime is {}", + device.getTableName(), + device, + deletion.getStartTime(), + deletion.getEndTime(), + fileStartTime, + fileEndTime); + } + if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime)) { + ++matchSize; + } else { + deletedByMods.add(sealedTsFile); + break; + } + } + if (matchSize == devicesInFile.size()) { + deletedByFiles.add(sealedTsFile); + } + + if (logger.isDebugEnabled()) { + logger.debug("expect is {}, actual is {}", devicesInFile.size(), matchSize); + for (TsFileResource tsFileResource : deletedByFiles) { + logger.debug( + "delete tsFileResource is {}", tsFileResource.getTsFile().getAbsolutePath()); + } + } + } else { + involvedModificationFiles.add(sealedTsFile.getModFileForWrite()); + } + } else { + involvedModificationFiles.add(sealedTsFile.getModFileForWrite()); + } + } + + for (TsFileResource tsFileResource : deletedByMods) { + if (tsFileResource.isClosed() + || !tsFileResource.getProcessor().deleteDataInMemory(deletion)) { + involvedModificationFiles.add(tsFileResource.getModFileForWrite()); + } // else do nothing + } + + if (!deletedByFiles.isEmpty()) { + deleteTsFileCompletely(deletedByFiles); + if (logger.isDebugEnabled()) { + logger.debug( + "deleteTsFileCompletely execute successful, all tsfile are deleted successfully"); + } } if (involvedModificationFiles.isEmpty()) { @@ -2984,6 +3082,27 @@ public class DataRegion implements IDataRegionForQuery { involvedModificationFiles.size()); } + private boolean isFileFullyMatchedByTime( + ModEntry deletion, long fileStartTime, long fileEndTime) { + return fileStartTime >= deletion.getStartTime() && fileEndTime <= deletion.getEndTime(); + } + + /** Delete completely TsFile and related supporting files */ + private void deleteTsFileCompletely(List<TsFileResource> tsfileResourceList) { + for (TsFileResource tsFileResource : tsfileResourceList) { + tsFileManager.remove(tsFileResource, tsFileResource.isSeq()); + tsFileResource.writeLock(); + try { + FileMetrics.getInstance() + .deleteTsFile(tsFileResource.isSeq(), Collections.singletonList(tsFileResource)); + tsFileResource.remove(); + logger.info("Remove tsfile {} directly when delete data", tsFileResource.getTsFilePath()); + } finally { + tsFileResource.writeUnlock(); + } + } + } + private void deleteDataDirectlyInFile(List<TsFileResource> tsfileResourceList, ModEntry modEntry) throws IOException { List<TsFileResource> deletedByMods = new ArrayList<>();
