jt2594838 commented on code in PR #19186:
URL: https://github.com/apache/iotdb/pull/19186#discussion_r4533475269


##########
integration-test/src/test/java/org/apache/iotdb/db/it/audit/IoTDBAuditLogBasicIT.java:
##########
@@ -2417,7 +2420,47 @@ public void basicAuditLogTestForTreeModel() throws 
SQLException, InterruptedExce
     connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT);
     statement = connection.createStatement();
     for (String sql : TREE_MODEL_AUDIT_SQLS_USER_ROOT_FINAL) {
-      statement.execute(sql);
+      if (sql.startsWith("delete timeseries ") || sql.startsWith("drop 
timeseries ")) {
+        final String paths = "[" + sql.substring(sql.lastIndexOf(' ') + 1) + 
"]";
+        // Poll the ConfigNode directly so diagnostics do not add SQL audit 
records to this test.
+        try (SyncConfigNodeIServiceClient client =
+            (SyncConfigNodeIServiceClient) 
EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+          final long previousProcedureId =
+              
client.showDeleteTimeSeriesProcedures().getProcedureInfoList().stream()
+                  .filter(p -> paths.equals(p.getPathPatterns()))
+                  .mapToLong(p -> p.getProcedureId())
+                  .max()
+                  .orElse(-1L);
+          statement.execute(sql);
+          await()
+              .atMost(60, TimeUnit.SECONDS)
+              .pollInSameThread()
+              .pollInterval(20, TimeUnit.MILLISECONDS)
+              .until(
+                  () -> {
+                    final java.util.Optional<
+                            
org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesProcedureInfo>
+                        procedure =
+                            
client.showDeleteTimeSeriesProcedures().getProcedureInfoList().stream()
+                                .filter(p -> paths.equals(p.getPathPatterns()))
+                                .filter(p -> p.getProcedureId() > 
previousProcedureId)
+                                .max(java.util.Comparator.comparingLong(p -> 
p.getProcedureId()));
+                    if (!procedure.isPresent()) {
+                      return false;
+                    }
+                    if ("ROLLEDBACK".equals(procedure.get().getState())) {
+                      throw new AssertionError(
+                          "Timeseries deletion procedure "
+                              + procedure.get().getProcedureId()
+                              + " rolled back: "
+                              + procedure.get().getErrorMessage());
+                    }
+                    return "SUCCESS".equals(procedure.get().getState());
+                  });
+        }

Review Comment:
   Why is TestUtils.deleteTimeSeries not applied here.
   And mind the import.



##########
integration-test/src/test/java/org/apache/iotdb/db/it/utils/IoTDBTestUtilsIT.java:
##########
@@ -63,6 +65,49 @@ public static void tearDown() {
     EnvFactory.getEnv().cleanClusterEnvironment();
   }
 
+  @Test
+  public void testColumnDeletionWaitsForFinalization() {
+    final AtomicInteger probes = new AtomicInteger();
+    final ResultSet absentColumn =
+        (ResultSet)
+            Proxy.newProxyInstance(
+                ResultSet.class.getClassLoader(),
+                new Class<?>[] {ResultSet.class},
+                (proxy, method, args) -> {
+                  if ("next".equals(method.getName())) {
+                    return false;
+                  }
+                  if ("close".equals(method.getName())) {
+                    return null;
+                  }
+                  throw new UnsupportedOperationException(method.getName());
+                });
+    final Statement statement =
+        (Statement)
+            Proxy.newProxyInstance(
+                Statement.class.getClassLoader(),
+                new Class<?>[] {Statement.class},
+                (proxy, method, args) -> {
+                  if ("executeQuery".equals(method.getName())) {
+                    return absentColumn;
+                  }
+                  if ("execute".equals(method.getName())) {
+                    assertEquals(
+                        "ALTER TABLE test.t DROP COLUMN IF EXISTS 
\"removed\"\"column\"", args[0]);
+                    if (probes.incrementAndGet() == 1) {
+                      throw new SQLException(
+                          "deletion still finalizing",
+                          null,
+                          
TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode());
+                    }
+                    return false;
+                  }
+                  throw new UnsupportedOperationException(method.getName());
+                });
+    TestUtils.waitForColumnDeletion(statement, "test.t", "removed\"column");
+    assertEquals(2, probes.get());
+  }

Review Comment:
   This does not seem to be an integration test.



##########
integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBNullableAttributeCacheIT.java:
##########
@@ -133,12 +135,37 @@ private void invalidationFailure(boolean loseAck) throws 
Exception {
         assertTrue(target.count(INVALIDATE) > 0);
       }
       target.release(hook);
-      SQLException failure = ddl.get(30, TimeUnit.SECONDS);
-      if (failure != null) {
-        assertInvalidationFailure(failure);
-        // DropTableColumnProcedure does not roll back its pre-delete marker. 
Reissuing the same
-        // drop is supported: preDeleteColumn adds that marker idempotently 
and completes the drop.
-        update("ALTER TABLE sensor DROP COLUMN changing");
+      assertNull("The asynchronous drop must be accepted", ddl.get(30, 
TimeUnit.SECONDS));
+      try (Connection connection = connection(remote);
+          Statement statement = connection.createStatement()) {
+        Awaitility.await()
+            .pollInterval(200, TimeUnit.MILLISECONDS)
+            .atMost(60, TimeUnit.SECONDS)
+            .until(
+                () -> {
+                  boolean present = false;
+                  try (ResultSet result =
+                      statement.executeQuery("DESCRIBE " + database + ".sensor 
DETAILS")) {
+                    while (result.next()) {
+                      present |= "changing".equals(result.getString(1));
+                    }
+                  }
+                  if (!present) {
+                    return true;
+                  }
+                  // A failed task retains its pre-delete marker. Retrying the 
drop after restoring
+                  // the RPC completes it; a still-running task rejects the 
overlapping request.
+                  try {
+                    update("ALTER TABLE sensor DROP COLUMN changing");
+                  } catch (SQLException e) {
+                    assertTrue(
+                        e.getMessage(),
+                        e.getErrorCode() == 
TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()
+                            || e.getErrorCode() == 
TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode());
+                  }
+                  return false;
+                });
+        TestUtils.waitForColumnDeletion(statement, database + ".sensor", 
"changing");

Review Comment:
   Are the two waits duplicated?



##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java:
##########
@@ -2515,92 +2542,14 @@ public TSStatus alterEncodingCompressor(final 
TAlterEncodingCompressorReq req) {
 
   @Override
   public TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req) {
-    TSStatus status = confirmLeader();
-    if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-      String queryId = req.getQueryId();
-      PathPatternTree rawPatternTree =
-          
PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree()));
-      boolean isGeneratedByPipe = req.isSetIsGeneratedByPipe() && 
req.isIsGeneratedByPipe();
-      /**
-       * If delete pattern is prefix path (such as root.db.**), it may be 
optimized to delete
-       * database plus create database. We need to determine two conditions: 
whether the pattern
-       * ends in **, and that the device is a full path and is matched in the 
ConfigMTree.
-       */
-      boolean canOptimize = false;
-      HashSet<TDatabaseSchema> deleteDatabaseSchemas = new HashSet<>();
-      List<PartialPath> deleteTimeSeriesPatternPaths = new ArrayList<>();
-      List<PartialPath> deleteDatabasePatternPaths = new ArrayList<>();
-      for (PartialPath path : rawPatternTree.getAllPathPatterns()) {
-        if (PathPatternUtil.isMultiLevelMatchWildcard(path.getMeasurement())
-            && !path.getDevicePath().hasWildcard()) {
-          Map<String, TDatabaseSchema> databaseSchemaMap =
-              
getClusterSchemaManager().getMatchedDatabaseSchemasByPrefix(path.getDevicePath());
-          // root.__audit can never be deleted
-          databaseSchemaMap.remove(TREE_MODEL_AUDIT_DATABASE);
-          if (!databaseSchemaMap.isEmpty()) {
-            deleteDatabaseSchemas.addAll(databaseSchemaMap.values());
-            deleteDatabasePatternPaths.add(path);
-            canOptimize = true;
-            continue;
-          }
-        }
-        deleteTimeSeriesPatternPaths.add(path);
-      }
-      if (!canOptimize) {
-        return procedureManager.deleteTimeSeries(
-            queryId,
-            rawPatternTree,
-            isGeneratedByPipe,
-            req.isSetMayDeleteAudit() && req.isMayDeleteAudit());
-      }
-      // check if the database is using template
-      try {
-        SchemaUtils.checkSchemaRegionUsingTemplate(this, 
deleteDatabasePatternPaths);
-      } catch (MetadataException e) {
-        return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
-      }
-      if (!deleteTimeSeriesPatternPaths.isEmpty()) {
-        // 1. delete time series that can not be optimized
-        PathPatternTree deleteTimeSeriesPatternTree = new PathPatternTree();
-        for (PartialPath path : deleteTimeSeriesPatternPaths) {
-          deleteTimeSeriesPatternTree.appendPathPattern(path);
-        }
-        deleteTimeSeriesPatternTree.constructTree();
-        status =
-            procedureManager.deleteTimeSeries(
-                queryId,
-                deleteTimeSeriesPatternTree,
-                isGeneratedByPipe,
-                req.isSetMayDeleteAudit() && req.isMayDeleteAudit());
-      }
-      if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-        // 2. delete database
-        List<TSStatus> failedStatus = new ArrayList<>();
-        status =
-            procedureManager.deleteDatabases(
-                new ArrayList<>(deleteDatabaseSchemas), isGeneratedByPipe);
-        if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-          failedStatus.add(status);
-        }
-        // 3. create database whatever the delete database operation is 
successful or not
-        for (TDatabaseSchema databaseSchema : deleteDatabaseSchemas) {
-          status =
-              clusterSchemaManager.setDatabase(
-                  new 
DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, databaseSchema),
-                  isGeneratedByPipe);
-          if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) 
{
-            failedStatus.add(status);
-          }
-        }
-        // 4. squash or generate status
-        if (!failedStatus.isEmpty()) {
-          status = RpcUtils.squashResponseStatusList(failedStatus);
-        } else {
-          status = StatusUtils.OK;
-        }
-      }
-    }
-    return status;
+    final TSStatus status = confirmLeader();
+    return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
+        ? procedureManager.deleteTimeSeries(
+            req.getQueryId(),
+            
PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree())),
+            req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe(),
+            req.isSetMayDeleteAudit() && req.isMayDeleteAudit())
+        : status;

Review Comment:
   May still preserve the optimizaitons and checks.



##########
integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java:
##########
@@ -1389,6 +1390,41 @@ public void testTreeViewTable() throws Exception {
     }
   }
 
+  @Test
+  public void testAsyncDropColumn() throws Exception {
+    try (Connection connection = 
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+        Statement statement = connection.createStatement();
+        Statement progress = TestUtils.createSingleNodeStatement(connection)) {
+      statement.execute("CREATE DATABASE async_drop_column_db");
+      try {
+        statement.execute("USE async_drop_column_db");
+        statement.execute(
+            "CREATE TABLE t (device STRING TAG, attr STRING ATTRIBUTE, removed 
INT32, kept INT32)");
+        statement.execute("INSERT INTO t(time,device,attr,removed,kept) 
VALUES(1,'d','a',11,21)");
+        statement.execute("FLUSH");
+        statement.execute("INSERT INTO t(time,device,attr,removed,kept) 
VALUES(2,'d','a',12,22)");
+        statement.execute("ALTER TABLE t DROP COLUMN removed");
+        TestUtils.waitForColumnDeletion(progress, "async_drop_column_db.t", 
"removed");
+        statement.execute("ALTER TABLE t ADD COLUMN removed STRING");
+        try (ResultSet result =
+            statement.executeQuery("SELECT removed,kept,attr FROM t ORDER BY 
time")) {
+          TestUtils.assertResultSetEqual(
+              result, "removed,kept,attr,", new String[] {"null,21,a,", 
"null,22,a,"});
+        }
+        // Attribute deletion has a different region/cache path from field 
deletion.
+        statement.execute("ALTER TABLE t DROP COLUMN attr");
+        TestUtils.waitForColumnDeletion(progress, "async_drop_column_db.t", 
"attr");

Review Comment:
   Why not TestUtils.dropColumn?



##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java:
##########
@@ -800,6 +809,39 @@ public int getLoadStatisticsPublisherThreadCount() {
     return loadStatisticsPublisherThreadCount;
   }
 
+  public int getTimeseriesDeletionQueueCapacity() {
+    return timeseriesDeletionQueueCapacity;
+  }
+
+  public void setTimeseriesDeletionQueueCapacity(int capacity) {
+    if (capacity <= 0) {
+      throw new IllegalArgumentException(CommonMessages.SIZE_MUST_BE_POSITIVE);
+    }
+    timeseriesDeletionQueueCapacity = capacity;
+  }
+
+  public long getTimeseriesDeletionExecutionIntervalMs() {
+    return timeseriesDeletionExecutionIntervalMs;
+  }
+
+  public void setTimeseriesDeletionExecutionIntervalMs(long intervalMs) {
+    if (intervalMs <= 0) {
+      throw new IllegalArgumentException(CommonMessages.SIZE_MUST_BE_POSITIVE);
+    }
+    timeseriesDeletionExecutionIntervalMs = intervalMs;
+  }

Review Comment:
   <= 0 means no wait



##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ConfigProcedureStore.java:
##########
@@ -71,12 +80,41 @@ public void setRunning(boolean running) {
 
   @Override
   public List<Procedure<ConfigNodeProcedureEnv>> load() {
-    return procedureInfo.oldLoad();
+    return copyDeletionProcedures(procedureInfo.oldLoad());
   }
 
   @Override
   public List<Procedure<ConfigNodeProcedureEnv>> getProcedures() {
-    return procedureInfo.getProcedures();
+    return copyDeletionProcedures(procedureInfo.getProcedures());
+  }
+
+  private List<Procedure<ConfigNodeProcedureEnv>> copyDeletionProcedures(
+      List<Procedure<ConfigNodeProcedureEnv>> procedures) {
+    final List<Procedure<ConfigNodeProcedureEnv>> copies = new 
ArrayList<>(procedures.size());
+    for (Procedure<ConfigNodeProcedureEnv> procedure : procedures) {
+      try {
+        copies.add(copyDeletionProcedure(procedure));
+      } catch (IOException e) {
+        throw new 
IllegalStateException(ConfigNodeMessages.LOAD_PROCEDURE_WAL_FAILED, e);
+      }
+    }
+    return copies;
+  }

Review Comment:
   Add a detailed explaination about why copying is necessary.



##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java:
##########
@@ -317,18 +440,55 @@ protected int getStateId(final DeleteTimeSeriesState 
deleteTimeSeriesState) {
 
   @Override
   protected DeleteTimeSeriesState getInitialState() {
-    return DeleteTimeSeriesState.CONSTRUCT_BLACK_LIST;
+    return DeleteTimeSeriesState.PREPARE_DELETE;
   }
 
   public String getQueryId() {
     return queryId;
   }
 
   public PathPatternTree getPatternTree() {
-    return patternTree;
+    return originalPatternTree == null ? patternTree : originalPatternTree;
+  }
+
+  public boolean deletesDatabase(final String database) {
+    // Table and tree databases have separate namespaces, even with the same 
qualified path.
+    if (PathUtils.isTableModelDatabase(database)) {
+      return false;
+    }
+    final PartialPath databasePath;
+    try {
+      databasePath = PartialPath.getQualifiedDatabasePartialPath(database);
+    } catch (IllegalPathException e) {
+      return false;
+    }
+    final String[] databaseNodes = databasePath.getNodes();
+    for (PartialPath path : getPatternTree().getAllPathPatterns()) {
+      if (!PathPatternUtil.isMultiLevelMatchWildcard(path.getMeasurement())
+          || path.getNodeLength() <= 1) {
+        continue;
+      }
+      final PartialPath prefix =
+          new PartialPath(Arrays.copyOf(path.getNodes(), path.getNodeLength() 
- 1));
+      if (!prefix.hasWildcard()
+          && prefix.getNodeLength() <= databaseNodes.length
+          && Arrays.equals(
+              prefix.getNodes(), Arrays.copyOf(databaseNodes, 
prefix.getNodeLength()))) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public String getDeleteProgress() {
+    final DeleteTimeSeriesState state = getCurrentState();
+    return isFinished() || state == null ? getState().name() : state.name();
   }

Review Comment:
   A detailed progress assembling dropping table?



##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java:
##########
@@ -2886,9 +3087,19 @@ public Pair<Long, Boolean> checkDuplicateTableTask(
     ProcedureType type;
     for (final Procedure<?> procedure : executor.getProcedures().values()) {
       type = ProcedureFactory.getProcedureType(procedure);
-      if (type == null || procedure.isFinished()) {
+      if (type == null
+          || (procedure.isFinished()
+              && !(procedure instanceof DropTableColumnProcedure
+                  && !(procedure instanceof DropViewColumnProcedure))
+              && !(procedure instanceof DeleteTimeSeriesProcedure))) {
         continue;
       }
+      if (procedure instanceof DeleteTimeSeriesProcedure
+          && (queryId == null
+              || !queryId.equals(((DeleteTimeSeriesProcedure) 
procedure).getQueryId()))
+          && ((DeleteTimeSeriesProcedure) 
procedure).deletesDatabase(database)) {
+        return new Pair<>(-1L, true);
+      }

Review Comment:
   Why would a table task involves DeleteTimeSeries



##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java:
##########
@@ -317,18 +440,55 @@ protected int getStateId(final DeleteTimeSeriesState 
deleteTimeSeriesState) {
 
   @Override
   protected DeleteTimeSeriesState getInitialState() {
-    return DeleteTimeSeriesState.CONSTRUCT_BLACK_LIST;
+    return DeleteTimeSeriesState.PREPARE_DELETE;
   }
 
   public String getQueryId() {
     return queryId;
   }
 
   public PathPatternTree getPatternTree() {
-    return patternTree;
+    return originalPatternTree == null ? patternTree : originalPatternTree;
+  }
+
+  public boolean deletesDatabase(final String database) {
+    // Table and tree databases have separate namespaces, even with the same 
qualified path.
+    if (PathUtils.isTableModelDatabase(database)) {
+      return false;
+    }
+    final PartialPath databasePath;
+    try {
+      databasePath = PartialPath.getQualifiedDatabasePartialPath(database);
+    } catch (IllegalPathException e) {
+      return false;
+    }
+    final String[] databaseNodes = databasePath.getNodes();
+    for (PartialPath path : getPatternTree().getAllPathPatterns()) {
+      if (!PathPatternUtil.isMultiLevelMatchWildcard(path.getMeasurement())
+          || path.getNodeLength() <= 1) {
+        continue;
+      }
+      final PartialPath prefix =
+          new PartialPath(Arrays.copyOf(path.getNodes(), path.getNodeLength() 
- 1));
+      if (!prefix.hasWildcard()
+          && prefix.getNodeLength() <= databaseNodes.length
+          && Arrays.equals(
+              prefix.getNodes(), Arrays.copyOf(databaseNodes, 
prefix.getNodeLength()))) {
+        return true;
+      }
+    }
+    return false;

Review Comment:
   Is it possible to search the database nodes directly in the pattern tree and 
check whether the matching node is a "\*\*" or contains a child of "\*\*", 
without constructing the intermediate paths?



##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java:
##########
@@ -147,6 +190,86 @@ protected Flow executeFromState(
     }
   }
 
+  /** Keep the full-database optimization inside the durable asynchronous 
deletion task. */
+  private void prepareDelete(final ConfigNodeProcedureEnv env) {
+    final ConfigManager manager = env.getConfigManager();
+    final Set<TDatabaseSchema> databases = new HashSet<>();
+    final List<PartialPath> databasePatterns = new ArrayList<>();
+    final PathPatternTree remaining = new PathPatternTree();
+    for (PartialPath path : patternTree.getAllPathPatterns()) {
+      if (PathPatternUtil.isMultiLevelMatchWildcard(path.getMeasurement())
+          && path.getNodeLength() > 1) {
+        // Pattern-tree paths are PartialPaths, whose getDevicePath() includes 
the final **.
+        final PartialPath prefix =
+            new PartialPath(Arrays.copyOf(path.getNodes(), 
path.getNodeLength() - 1));
+        if (prefix.hasWildcard()) {
+          remaining.appendPathPattern(path);
+          continue;
+        }
+        final Map<String, TDatabaseSchema> matched =
+            
manager.getClusterSchemaManager().getMatchedDatabaseSchemasByPrefix(prefix);

Review Comment:
   May still delete a database if it is fully covered by the prefix.



##########
iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template:
##########
@@ -2381,6 +2381,22 @@ procedure_completed_clean_interval=30
 # Datatype: int
 procedure_completed_evict_ttl=60
 
+# Maximum number of tree-model timeseries deletion tasks waiting to execute on 
the ConfigNode.
+# effectiveMode: restart
+# Datatype: int, greater than 0
+timeseries_deletion_queue_capacity=1024
+
+# Minimum interval between starting tree-model timeseries deletion tasks, in 
milliseconds.
+# Measured when tasks leave the queue to start execution; submissions are not 
rate-limited.
+# effectiveMode: restart
+# Datatype: long, greater than 0
+timeseries_deletion_execution_interval_ms=1000

Review Comment:
   Make it hot-reload



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to