This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch optimize_view_deletion in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f9645ac5f97ac8cf47d3303bc960dec3b70c67b5 Author: Tian Jiang <[email protected]> AuthorDate: Thu Jan 9 10:28:20 2025 +0800 Do not add the source path of a view when it can be matched by the deletion pattern (cherry picked from commit 6ca6a432e798c8fa313b9bf71e58ffae54f9e9a4) --- .../org/apache/iotdb/db/it/IoTDBDeletionIT.java | 67 ++++++++++++++++++++++ .../queryengine/plan/analyze/AnalyzeVisitor.java | 12 ++++ .../org/apache/iotdb/commons/utils/FileUtils.java | 27 +++++++++ 3 files changed, 106 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java index 576b9c75f21..06e69b79aaa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDeletionIT.java @@ -19,7 +19,11 @@ package org.apache.iotdb.db.it; +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -32,10 +36,13 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.io.File; +import java.io.IOException; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; import java.util.Locale; import static org.junit.Assert.assertEquals; @@ -467,6 +474,66 @@ public class IoTDBDeletionIT { } } + @Test + public void testDeleteWithView() throws SQLException, IOException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO root.del_with_view.d1(timestamp, status) VALUES(1, 1)"); + statement.execute("INSERT INTO root.del_with_view.d2(timestamp, status) VALUES(2, 2)"); + statement.execute("INSERT INTO root.del_with_view.d3(timestamp, status) VALUES(3, 3)"); + statement.execute("INSERT INTO root.del_with_view.d4(timestamp, status) VALUES(4, 4)"); + + statement.execute( + "CREATE VIEW root.del_with_view.d1_view.status as root.del_with_view.d1.status"); + statement.execute( + "CREATE VIEW root.del_with_view.d2_view.status as root.del_with_view.d2.status"); + statement.execute( + "CREATE VIEW root.del_with_view.d3_view.status as root.del_with_view.d3.status"); + statement.execute( + "CREATE VIEW root.del_with_view.d4_view.status as root.del_with_view.d4.status"); + + statement.execute("FLUSH"); + + try (ResultSet resultSet = + statement.executeQuery("select status from root.del_with_view.*")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(4, cnt); + } + + statement.execute("DELETE FROM root.del_with_view.*.status"); + + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String nodePropPath = dataNodeWrapper.getSystemPropertiesPath(); + String nodeDatabasePath = nodePropPath + "../../../data/sequence/root.del_with_view"; + File nodeDatabaseFile = new File(nodeDatabasePath); + List<File> modFiles = + FileUtils.listFilesRecursively( + nodeDatabaseFile, f -> f.getName().endsWith(ModificationFile.FILE_SUFFIX)); + for (File modFile : modFiles) { + List<ModEntry> allMods; + try (ModificationFile modificationFile = new ModificationFile(modFile)) { + allMods = modificationFile.getAllMods(); + } + // the source paths of the views can be matched by the pattern, so they should not appear + // in the mod file + assertEquals(1, allMods.size()); + } + } + + try (ResultSet resultSet = + statement.executeQuery("select status from root.del_with_view.*")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + } + Assert.assertEquals(0, cnt); + } + } + } + private static void prepareSeries() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index aa6653da70f..156ceb5a73b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -3521,6 +3521,18 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> logicalViewSchema = (LogicalViewSchema) measurementSchema; if (logicalViewSchema.isWritable()) { sourcePathOfAliasSeries = logicalViewSchema.getSourcePathIfWritable(); + // if the source path can be matched by any of the deletion pattern, do not add it + boolean pathMatched = false; + for (MeasurementPath deletionPattern : deleteDataStatement.getPathList()) { + if (deletionPattern.matchFullPath(sourcePathOfAliasSeries)) { + pathMatched = true; + break; + } + } + if (pathMatched) { + continue; + } + deletePatternSet.add(new MeasurementPath(sourcePathOfAliasSeries.getNodes())); deduplicatedDeviceIDs.add(sourcePathOfAliasSeries.getIDeviceID()); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java index f1ae48c9956..fbc0621caaa 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java @@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileFilter; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -41,7 +42,10 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.text.CharacterIterator; import java.text.StringCharacterIterator; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.Stack; public class FileUtils { private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); @@ -50,6 +54,29 @@ public class FileUtils { private FileUtils() {} + public static List<File> listFilesRecursively(File dir, FileFilter fileFilter) { + List<File> result = new ArrayList<>(); + Stack<File> stack = new Stack<>(); + if (dir.exists()) { + stack.push(dir); + } + while (!stack.isEmpty()) { + File file = stack.pop(); + if (file.isDirectory()) { + File[] files = file.listFiles(); + if (files != null) { + for (File f : files) { + stack.push(f); + } + } + } + if (fileFilter.accept(file)) { + result.add(file); + } + } + return result; + } + public static boolean deleteFileIfExist(File file) { try { Files.deleteIfExists(file.toPath());
