This is an automated email from the ASF dual-hosted git repository.

deardeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 31a0185eb47 [fix](fe) Fix publish recovery and task cleanup after 
backend outages (#70181)
31a0185eb47 is described below

commit 31a0185eb473b22241f55a9996eeea996afdf897
Author: deardeng <[email protected]>
AuthorDate: Tue Nov 17 15:27:53 2026 +0800

    [fix](fe) Fix publish recovery and task cleanup after backend outages 
(#70181)
    
    Related PR: #57591
    
    #57591 introduced `AgentTaskCleanupDaemon`, which removes queued
    `PUBLISH_VERSION` tasks after three consecutive offline checks. The
    transaction still records that its publish tasks were sent, so FE skips
    regenerating them. When the BE recovers, its task report finds no task
    to resend, and tablet-report republishing cannot help because the
    undelivered task has no failed tablets recorded. As a result, the
    transaction can remain `COMMITTED` and invisible after the BE recovers.
    
    Keep publish tasks retryable during temporary outages. When a backend is
    permanently removed (including replay), abort and reclaim all its queued
    tasks. When a database is finally erased, reclaim that database's
    publish tasks across backends.
    
    Publish admission and terminal cleanup share the queue lock. Lock-free
    backend/database lookups and the database's dropped marker prevent
    in-flight producers from recreating tasks after cleanup. The existing
    abort callback and synchronous-delete notification are reused, and task
    counts reflect the removals.
    
    Publish producers skip dispatch for rejected tasks. The publisher still
    tracks an explicit aborted task for a removed backend so absence is not
    interpreted as successful publication. Repeated report-driven requests
    can contain different partition subsets and retain their original
    dispatch behavior even when the transaction signature is already queued;
    only explicitly ABORTED tasks skip dispatch.
---
 .../main/java/org/apache/doris/catalog/Env.java    |   2 +
 .../org/apache/doris/master/ReportHandler.java     |   6 +-
 .../org/apache/doris/system/SystemInfoService.java |   3 +
 .../apache/doris/task/AgentTaskCleanupDaemon.java  |  17 +-
 .../java/org/apache/doris/task/AgentTaskQueue.java |  63 ++-
 .../doris/transaction/PublishVersionDaemon.java    |   8 +-
 .../doris/task/AgentTaskCleanupDaemonTest.java     | 441 +++++++++++++++++++++
 7 files changed, 515 insertions(+), 25 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java 
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 7f17dcbca82..a8a7b686227 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -291,6 +291,7 @@ import org.apache.doris.system.SystemInfoService.HostInfo;
 import org.apache.doris.task.AgentBatchTask;
 import org.apache.doris.task.AgentTaskCleanupDaemon;
 import org.apache.doris.task.AgentTaskExecutor;
+import org.apache.doris.task.AgentTaskQueue;
 import org.apache.doris.task.CompactionTask;
 import org.apache.doris.task.MasterTaskExecutor;
 import org.apache.doris.task.PriorityMasterTaskExecutor;
@@ -7466,6 +7467,7 @@ public class Env {
     public void eraseDatabase(long dbId, boolean needEditLog) {
         // remove database transaction manager
         
Env.getCurrentGlobalTransactionMgr().removeDatabaseTransactionMgr(dbId);
+        AgentTaskQueue.removePublishVersionTasks(dbId);
 
         if (needEditLog) {
             Env.getCurrentEnv().getEditLog().logEraseDb(dbId);
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java 
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 2137dd6cce8..ea9828962cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -1338,9 +1338,11 @@ public class ReportHandler extends Daemon {
             for (long txnId : map.keySet()) {
                 PublishVersionTask task = new PublishVersionTask(backendId, 
txnId, dbId,
                         Lists.newArrayList(map.get(txnId)), 
createPublishVersionTaskTime);
-                batchTask.addTask(task);
-                // add to AgentTaskQueue for handling finish report.
+                // A duplicate signature can carry new partitions; only 
terminally aborted tasks are skipped.
                 AgentTaskQueue.addTask(task);
+                if (task.getErrorCode() != TStatusCode.ABORTED) {
+                    batchTask.addTask(task);
+                }
             }
         }
         AgentTaskExecutor.submit(batchTask);
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java 
b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
index aa313f7bb1c..b5c89a3758e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
@@ -40,6 +40,7 @@ import 
org.apache.doris.nereids.trees.plans.commands.info.ModifyBackendHostNameO
 import org.apache.doris.nereids.trees.plans.commands.info.ModifyBackendOp;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.resource.Tag;
+import org.apache.doris.task.AgentTaskQueue;
 import org.apache.doris.thrift.TStatusCode;
 import org.apache.doris.thrift.TStorageMedium;
 
@@ -300,6 +301,7 @@ public class SystemInfoService {
         copiedBackends.remove(droppedBackend.getId());
         ImmutableMap<Long, Backend> newIdToBackend = 
ImmutableMap.copyOf(copiedBackends);
         idToBackendRef = newIdToBackend;
+        AgentTaskQueue.removeBackendTasks(droppedBackend.getId());
 
         // update idToReportVersion
         Map<Long, AtomicLong> copiedReportVersions = 
Maps.newHashMap(idToReportVersionRef);
@@ -850,6 +852,7 @@ public class SystemInfoService {
         copiedBackends.remove(backend.getId());
         ImmutableMap<Long, Backend> newIdToBackend = 
ImmutableMap.copyOf(copiedBackends);
         idToBackendRef = newIdToBackend;
+        AgentTaskQueue.removeBackendTasks(backend.getId());
 
         // update idToReportVersion
         Map<Long, AtomicLong> copiedReportVersions = 
Maps.newHashMap(idToReportVersionRef);
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/task/AgentTaskCleanupDaemon.java 
b/fe/fe-core/src/main/java/org/apache/doris/task/AgentTaskCleanupDaemon.java
index 23c1d2ee7fc..08ea7c4eaef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/AgentTaskCleanupDaemon.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/AgentTaskCleanupDaemon.java
@@ -19,10 +19,8 @@ package org.apache.doris.task;
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.Status;
 import org.apache.doris.common.util.MasterDaemon;
 import org.apache.doris.system.SystemInfoService;
-import org.apache.doris.thrift.TStatusCode;
 
 import com.google.common.collect.Maps;
 import org.apache.logging.log4j.LogManager;
@@ -66,18 +64,7 @@ public class AgentTaskCleanupDaemon extends MasterDaemon {
     }
 
     private void removeInactiveBeAgentTasks(Long beId) {
-        AgentTaskQueue.removeTask(beId, (agentTask -> {
-            String errMsg = "BE down, this agent task is aborted";
-            if (agentTask instanceof PushTask) {
-                PushTask task = ((PushTask) agentTask);
-                task.countDownLatchWithStatus(beId, agentTask.getTabletId(), 
new Status(TStatusCode.ABORTED, errMsg));
-            }
-            agentTask.setFinished(true);
-            agentTask.setErrorCode(TStatusCode.ABORTED);
-            agentTask.setErrorMsg(errMsg);
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("BE down, remove agent task: {}", agentTask);
-            }
-        }));
+        AgentTaskQueue.removeTask(beId,
+                task -> AgentTaskQueue.abortTask(task, "BE down, this agent 
task is aborted"));
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AgentTaskQueue.java 
b/fe/fe-core/src/main/java/org/apache/doris/task/AgentTaskQueue.java
index b61a3aa708d..26c746fdcac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/AgentTaskQueue.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/AgentTaskQueue.java
@@ -17,7 +17,11 @@
 
 package org.apache.doris.task;
 
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.Status;
 import org.apache.doris.thrift.TPushType;
+import org.apache.doris.thrift.TStatusCode;
 import org.apache.doris.thrift.TTaskType;
 
 import com.google.common.collect.HashBasedTable;
@@ -54,6 +58,14 @@ public class AgentTaskQueue {
     public static synchronized boolean addTask(AgentTask task) {
         long backendId = task.getBackendId();
         TTaskType type = task.getTaskType();
+        if (type == TTaskType.PUBLISH_VERSION) {
+            // These lock-free metadata reads serialize late arrivals with 
terminal queue cleanup.
+            Database db = 
Env.getCurrentInternalCatalog().getDbNullable(task.getDbId());
+            if (Env.getCurrentSystemInfo().getBackend(backendId) == null || db 
== null || db.isDropped()) {
+                abortTask(task, "Backend or database has been removed");
+                return false;
+            }
+        }
 
         Map<Long, AgentTask> signatureMap = tasks.get(backendId, type);
         if (signatureMap == null) {
@@ -98,17 +110,58 @@ public class AgentTaskQueue {
     }
 
     public static synchronized void removeTask(long backendId, 
Consumer<AgentTask> onTaskRemoved) {
-        Map<TTaskType, Map<Long, AgentTask>> tasks = 
AgentTaskQueue.tasks.row(backendId);
-        tasks.forEach((type, taskSet) -> {
-            Iterator<Map.Entry<Long, AgentTask>> it = 
taskSet.entrySet().iterator();
+        removeTask(backendId, onTaskRemoved, true);
+    }
+
+    public static synchronized void removeBackendTasks(long backendId) {
+        removeTask(backendId, task -> abortTask(task, "BE has been removed"), 
false);
+        tasks.row(backendId).clear();
+    }
+
+    public static synchronized void removePublishVersionTasks(long dbId) {
+        for (Map<Long, AgentTask> taskSet : 
tasks.column(TTaskType.PUBLISH_VERSION).values()) {
+            Iterator<AgentTask> it = taskSet.values().iterator();
             while (it.hasNext()) {
-                Map.Entry<Long, AgentTask> entry = it.next();
+                AgentTask task = it.next();
+                if (task.getDbId() == dbId) {
+                    it.remove();
+                    --taskNum;
+                    abortTask(task, "Database has been removed");
+                }
+            }
+        }
+    }
+
+    private static void removeTask(long backendId, Consumer<AgentTask> 
onTaskRemoved, boolean keepPublishTasks) {
+        Map<TTaskType, Map<Long, AgentTask>> backendTasks = 
tasks.row(backendId);
+        backendTasks.forEach((type, taskSet) -> {
+            if (keepPublishTasks && type == TTaskType.PUBLISH_VERSION) {
+                // Keep publish tasks retryable during temporary backend 
outages.
+                return;
+            }
+            Iterator<AgentTask> it = taskSet.values().iterator();
+            while (it.hasNext()) {
+                AgentTask task = it.next();
                 it.remove();
-                onTaskRemoved.accept(entry.getValue());
+                --taskNum;
+                onTaskRemoved.accept(task);
             }
         });
     }
 
+    static void abortTask(AgentTask task, String errMsg) {
+        if (task instanceof PushTask) {
+            ((PushTask) task).countDownLatchWithStatus(task.getBackendId(), 
task.getTabletId(),
+                    new Status(TStatusCode.ABORTED, errMsg));
+        }
+        task.setFinished(true);
+        task.setErrorCode(TStatusCode.ABORTED);
+        task.setErrorMsg(errMsg);
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("abort agent task: {}", task);
+        }
+    }
+
     /*
      * we cannot define a push task with only 'backendId', 'signature' and 
'TTaskType'
      * add version and TPushType to help
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
 
b/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
index fcbd6705ac2..b10d7899531 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
@@ -37,6 +37,7 @@ import org.apache.doris.task.AgentTaskQueue;
 import org.apache.doris.task.PublishVersionTask;
 import org.apache.doris.task.UpdateVisibleVersionTask;
 import org.apache.doris.thrift.TPartitionVersionInfo;
+import org.apache.doris.thrift.TStatusCode;
 import org.apache.doris.thrift.TTaskType;
 
 import com.google.common.collect.Maps;
@@ -438,11 +439,12 @@ public class PublishVersionDaemon extends MasterDaemon {
                     partitionVersionInfos,
                     createPublishVersionTaskTime);
             task.setBaseTabletsIds(beIdToBaseTabletIds.getOrDefault(backendId, 
Collections.emptySet()));
-            // add to AgentTaskQueue for handling finish report.
-            // not check return value, because the add will success
+            // Keep rejected tasks in the transaction so a removed backend is 
not treated as a success.
             AgentTaskQueue.addTask(task);
-            batchTask.addTask(task);
             transactionState.addPublishVersionTask(backendId, task);
+            if (task.getErrorCode() != TStatusCode.ABORTED) {
+                batchTask.addTask(task);
+            }
         }
     }
 }
diff --git 
a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskCleanupDaemonTest.java
 
b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskCleanupDaemonTest.java
new file mode 100644
index 00000000000..60ba195fb00
--- /dev/null
+++ 
b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskCleanupDaemonTest.java
@@ -0,0 +1,441 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.task;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.Status;
+import org.apache.doris.common.jmockit.Deencapsulation;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.master.ReportHandler;
+import org.apache.doris.metric.MetricRepo;
+import org.apache.doris.persist.EditLog;
+import org.apache.doris.system.Backend;
+import org.apache.doris.system.SystemInfoService;
+import org.apache.doris.thrift.TPartitionVersionInfo;
+import org.apache.doris.thrift.TStatusCode;
+import org.apache.doris.thrift.TTaskType;
+import org.apache.doris.transaction.GlobalTransactionMgrIface;
+import org.apache.doris.transaction.PublishVersionDaemon;
+import org.apache.doris.transaction.TransactionState;
+import org.apache.doris.transaction.TransactionStatus;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.SetMultimap;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+public class AgentTaskCleanupDaemonTest {
+    private static final long BACKEND_ID = 10001L;
+    private static final long HEALTHY_BACKEND_ID = 10002L;
+    private static final long TRANSACTION_ID = 99L;
+
+    private MockedStatic<Env> mockedEnv;
+    private Backend backend;
+    private AgentTaskCleanupDaemon daemon;
+    private SystemInfoService infoService;
+    private InternalCatalog catalog;
+    private Database database;
+    private Env env;
+    private GlobalTransactionMgrIface transactionMgr;
+
+    @BeforeEach
+    public void setUp() {
+        MetricRepo.init();
+        AgentTaskQueue.clearAllTasks();
+        backend = new Backend(BACKEND_ID, "127.0.0.1", 9050);
+        backend.setAlive(false);
+        Backend healthyBackend = new Backend(HEALTHY_BACKEND_ID, "127.0.0.2", 
9050);
+        healthyBackend.setAlive(true);
+        infoService = new SystemInfoService();
+        infoService.addBackend(backend);
+        infoService.addBackend(healthyBackend);
+        catalog = Mockito.mock(InternalCatalog.class);
+        database = Mockito.spy(new Database(1L, "publish_test"));
+        Mockito.when(catalog.getDbNullable(1L)).thenReturn(database);
+        Mockito.when(catalog.getDbNullable(2L)).thenReturn(new Database(2L, 
"other_publish_test"));
+        env = Mockito.mock(Env.class);
+        Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class));
+        transactionMgr = Mockito.mock(GlobalTransactionMgrIface.class);
+        Mockito.doCallRealMethod().when(env).eraseDatabase(Mockito.anyLong(), 
Mockito.anyBoolean());
+        mockedEnv = mockEnv();
+        daemon = new AgentTaskCleanupDaemon();
+    }
+
+    @AfterEach
+    public void tearDown() {
+        mockedEnv.close();
+        AgentTaskQueue.clearAllTasks();
+    }
+
+    @Test
+    public void testOfflineCleanupPreservesPublishTasksForResend() {
+        PublishVersionTask publishTask = newPublishTask(BACKEND_ID, 
TRANSACTION_ID);
+        PublishVersionTask anotherPublishTask = newPublishTask(BACKEND_ID, 
TRANSACTION_ID + 1);
+        DropReplicaTask dropTask = new DropReplicaTask(BACKEND_ID, 10L, 20L, 
0, false);
+        DropReplicaTask healthyDropTask = new 
DropReplicaTask(HEALTHY_BACKEND_ID, 11L, 21L, 0, false);
+        AgentTaskQueue.addTask(publishTask);
+        AgentTaskQueue.addTask(anotherPublishTask);
+        AgentTaskQueue.addTask(dropTask);
+        AgentTaskQueue.addTask(healthyDropTask);
+
+        for (int i = 0; i < AgentTaskCleanupDaemon.MAX_FAILURE_TIMES - 1; i++) 
{
+            daemon.runAfterCatalogReady();
+            Assertions.assertTrue(AgentTaskQueue.contains(dropTask));
+            Assertions.assertFalse(dropTask.isFinished());
+        }
+        daemon.runAfterCatalogReady();
+        Assertions.assertFalse(AgentTaskQueue.contains(dropTask));
+        Assertions.assertTrue(dropTask.isFinished());
+        Assertions.assertEquals(TStatusCode.ABORTED, dropTask.getErrorCode());
+        Assertions.assertTrue(AgentTaskQueue.contains(healthyDropTask));
+        Assertions.assertFalse(healthyDropTask.isFinished());
+
+        // Further offline checks must not discard or finish the retained 
publish tasks either.
+        daemon.runAfterCatalogReady();
+        for (PublishVersionTask task : List.of(publishTask, 
anotherPublishTask)) {
+            Assertions.assertSame(task, AgentTaskQueue.getTask(BACKEND_ID, 
TTaskType.PUBLISH_VERSION,
+                    task.getSignature()));
+            Assertions.assertFalse(task.isFinished());
+            Assertions.assertNull(task.getErrorCode());
+            Assertions.assertTrue(task.getErrorTablets().isEmpty());
+        }
+
+        backend.setAlive(true);
+        daemon.runAfterCatalogReady();
+        try (MockedStatic<AgentTaskExecutor> executor = 
Mockito.mockStatic(AgentTaskExecutor.class)) {
+            Deencapsulation.invoke(ReportHandler.class, "taskReport", 
BACKEND_ID, Collections.emptyMap(), 0L);
+            ArgumentCaptor<AgentBatchTask> batch = 
ArgumentCaptor.forClass(AgentBatchTask.class);
+            executor.verify(() -> AgentTaskExecutor.submit(batch.capture()));
+            Assertions.assertEquals(Set.of(publishTask, anotherPublishTask),
+                    Set.copyOf(batch.getValue().getAllTasks()));
+
+            executor.reset();
+            Map<TTaskType, Set<Long>> runningTasks = 
Map.of(TTaskType.PUBLISH_VERSION,
+                    Set.of(publishTask.getSignature(), 
anotherPublishTask.getSignature()));
+            Deencapsulation.invoke(ReportHandler.class, "taskReport", 
BACKEND_ID, runningTasks, 2L);
+            executor.verifyNoInteractions();
+        }
+    }
+
+    @Test
+    public void testOfflineCleanupPreservesPartialPublishResult() {
+        PublishVersionTask task = newPublishTask(BACKEND_ID, TRANSACTION_ID);
+        task.setSuccTablets(Map.of(10L, 3L));
+        task.addErrorTablets(List.of(11L));
+        task.setErrorCode(TStatusCode.RUNTIME_ERROR);
+        task.setFinished(true);
+        AgentTaskQueue.addTask(task);
+
+        for (int i = 0; i < AgentTaskCleanupDaemon.MAX_FAILURE_TIMES; i++) {
+            daemon.runAfterCatalogReady();
+        }
+        Assertions.assertSame(task, AgentTaskQueue.getTask(BACKEND_ID, 
TTaskType.PUBLISH_VERSION,
+                task.getSignature()));
+        Assertions.assertTrue(task.isFinished());
+        Assertions.assertEquals(TStatusCode.RUNTIME_ERROR, 
task.getErrorCode());
+        Assertions.assertEquals(Map.of(10L, 3L), task.getSuccTablets());
+        Assertions.assertEquals(List.of(11L), task.getErrorTablets());
+        Assertions.assertEquals(List.of(task), 
AgentTaskQueue.getDiffTasks(BACKEND_ID, Collections.emptyMap()));
+    }
+
+    @Test
+    public void testVisibleTransactionRemovesRetainedOfflinePublishTask() 
throws Exception {
+        TransactionState transaction = newTransaction();
+        transaction.setTransactionStatus(TransactionStatus.COMMITTED);
+        PublishVersionTask offlineTask = newPublishTask(BACKEND_ID, 
transaction.getTransactionId());
+        PublishVersionTask healthyTask = newPublishTask(HEALTHY_BACKEND_ID, 
transaction.getTransactionId());
+        transaction.addPublishVersionTask(BACKEND_ID, offlineTask);
+        transaction.addPublishVersionTask(HEALTHY_BACKEND_ID, healthyTask);
+        transaction.setSendedTask();
+        AgentTaskQueue.addTask(offlineTask);
+        AgentTaskQueue.addTask(healthyTask);
+        for (int i = 0; i < AgentTaskCleanupDaemon.MAX_FAILURE_TIMES; i++) {
+            daemon.runAfterCatalogReady();
+        }
+        Assertions.assertTrue(AgentTaskQueue.contains(offlineTask));
+        Assertions.assertFalse(offlineTask.isFinished());
+        Assertions.assertTrue(transaction.hasSendTask());
+
+        GlobalTransactionMgrIface transactionMgr = 
Mockito.mock(GlobalTransactionMgrIface.class);
+        Mockito.doAnswer(invocation -> {
+            transaction.setTransactionStatus(TransactionStatus.VISIBLE);
+            return null;
+        
}).when(transactionMgr).finishTransaction(Mockito.eq(transaction.getDbId()),
+                Mockito.eq(transaction.getTransactionId()), Mockito.anyMap(), 
Mockito.anyMap());
+
+        // Exercise the publisher's existing terminal cleanup without 
recovering the offline BE.
+        Deencapsulation.invoke(new PublishVersionDaemon(), "tryFinishTxnSync", 
transaction, transactionMgr);
+        Assertions.assertFalse(backend.isAlive());
+        Assertions.assertEquals(TransactionStatus.VISIBLE, 
transaction.getTransactionStatus());
+        
Assertions.assertTrue(AgentTaskQueue.getTask(TTaskType.PUBLISH_VERSION).isEmpty());
+        Assertions.assertTrue(transaction.getPublishVersionTasks().isEmpty());
+        Assertions.assertEquals(0, AgentTaskQueue.getTaskNum());
+    }
+
+    @Test
+    public void testDropBackendReclaimsReportPublishAndOtherTasks() throws 
Exception {
+        try (MockedStatic<AgentTaskExecutor> executor = 
Mockito.mockStatic(AgentTaskExecutor.class)) {
+            Deencapsulation.invoke(ReportHandler.class, 
"handleRepublishVersionInfo", republishInfo(), BACKEND_ID);
+        }
+        PublishVersionTask task = (PublishVersionTask) AgentTaskQueue.getTask(
+                BACKEND_ID, TTaskType.PUBLISH_VERSION, TRANSACTION_ID);
+        Assertions.assertNotNull(task);
+        for (int i = 0; i < AgentTaskCleanupDaemon.MAX_FAILURE_TIMES; i++) {
+            daemon.runAfterCatalogReady();
+        }
+        Assertions.assertTrue(AgentTaskQueue.contains(task));
+
+        DropReplicaTask dropTask = new DropReplicaTask(BACKEND_ID, 10L, 20L, 
0, false);
+        PushTask pushTask = Mockito.mock(PushTask.class);
+        Mockito.when(pushTask.getBackendId()).thenReturn(BACKEND_ID);
+        
Mockito.when(pushTask.getTaskType()).thenReturn(TTaskType.REALTIME_PUSH);
+        AgentTaskQueue.addTask(dropTask);
+        AgentTaskQueue.addTask(pushTask);
+        PublishVersionTask healthyTask = newPublishTask(HEALTHY_BACKEND_ID, 
TRANSACTION_ID);
+        AgentTaskQueue.addTask(healthyTask);
+
+        infoService.dropBackend(BACKEND_ID);
+        Assertions.assertNull(infoService.getBackend(BACKEND_ID));
+        Assertions.assertFalse(AgentTaskQueue.contains(task));
+        Assertions.assertTrue(task.isFinished());
+        Assertions.assertEquals(TStatusCode.ABORTED, task.getErrorCode());
+        Assertions.assertFalse(AgentTaskQueue.contains(dropTask));
+        Assertions.assertTrue(dropTask.isFinished());
+        
Mockito.verify(pushTask).countDownLatchWithStatus(Mockito.eq(BACKEND_ID), 
Mockito.eq(0L),
+                Mockito.any(Status.class));
+        Mockito.verify(pushTask).setFinished(true);
+        Assertions.assertTrue(AgentTaskQueue.contains(healthyTask));
+        Assertions.assertEquals(1, AgentTaskQueue.getTaskNum());
+    }
+
+    @Test
+    public void testReplayDropBackendReclaimsPublishTask() {
+        PublishVersionTask task = newPublishTask(BACKEND_ID, TRANSACTION_ID);
+        AgentTaskQueue.addTask(task);
+        infoService.replayDropBackend(backend);
+        Assertions.assertFalse(AgentTaskQueue.contains(task));
+        Assertions.assertTrue(task.isFinished());
+        Assertions.assertEquals(TStatusCode.ABORTED, task.getErrorCode());
+        Assertions.assertEquals(0, AgentTaskQueue.getTaskNum());
+    }
+
+    @Test
+    public void testEraseDatabaseReclaimsOnlyItsPublishTasks() {
+        PublishVersionTask task = newPublishTask(BACKEND_ID, TRANSACTION_ID);
+        PublishVersionTask anotherBackendTask = 
newPublishTask(HEALTHY_BACKEND_ID, TRANSACTION_ID);
+        PublishVersionTask otherDbTask = new PublishVersionTask(BACKEND_ID, 
TRANSACTION_ID + 1, 2L,
+                List.of(new TPartitionVersionInfo(2L, 3L, 0L)), 0L);
+        DropReplicaTask dropTask = new DropReplicaTask(BACKEND_ID, 10L, 20L, 
0, false);
+        AgentTaskQueue.addTask(task);
+        AgentTaskQueue.addTask(anotherBackendTask);
+        AgentTaskQueue.addTask(otherDbTask);
+        AgentTaskQueue.addTask(dropTask);
+
+        database.markDropped();
+        env.eraseDatabase(1L, false);
+        Mockito.verify(transactionMgr).removeDatabaseTransactionMgr(1L);
+        for (PublishVersionTask removed : List.of(task, anotherBackendTask)) {
+            Assertions.assertFalse(AgentTaskQueue.contains(removed));
+            Assertions.assertTrue(removed.isFinished());
+            Assertions.assertEquals(TStatusCode.ABORTED, 
removed.getErrorCode());
+        }
+        Assertions.assertTrue(AgentTaskQueue.contains(otherDbTask));
+        Assertions.assertFalse(otherDbTask.isFinished());
+        Assertions.assertTrue(AgentTaskQueue.contains(dropTask));
+        Assertions.assertFalse(dropTask.isFinished());
+        Assertions.assertEquals(2, AgentTaskQueue.getTaskNum());
+    }
+
+    @Test
+    public void testLateReportAfterBackendDropCannotRecreateTask() throws 
Exception {
+        CountDownLatch reportPrepared = new CountDownLatch(1);
+        CountDownLatch resumeReport = new CountDownLatch(1);
+        ExecutorService worker = Executors.newSingleThreadExecutor();
+        Map<Long, SetMultimap<Long, TPartitionVersionInfo>> report = 
republishInfo();
+        try {
+            Future<?> future = worker.submit(() -> {
+                try (MockedStatic<Env> threadEnv = mockEnv();
+                        MockedStatic<AgentTaskExecutor> executor = 
Mockito.mockStatic(AgentTaskExecutor.class)) {
+                    reportPrepared.countDown();
+                    Assertions.assertTrue(resumeReport.await(10, 
TimeUnit.SECONDS));
+                    Deencapsulation.invoke(ReportHandler.class, 
"handleRepublishVersionInfo", report, BACKEND_ID);
+                    ArgumentCaptor<AgentBatchTask> batch = 
ArgumentCaptor.forClass(AgentBatchTask.class);
+                    executor.verify(() -> 
AgentTaskExecutor.submit(batch.capture()));
+                    
Assertions.assertTrue(batch.getValue().getAllTasks().isEmpty());
+                }
+                return null;
+            });
+            Assertions.assertTrue(reportPrepared.await(10, TimeUnit.SECONDS));
+            infoService.dropBackend(BACKEND_ID);
+            resumeReport.countDown();
+            future.get(10, TimeUnit.SECONDS);
+            Assertions.assertEquals(0, AgentTaskQueue.getTaskNum());
+        } finally {
+            resumeReport.countDown();
+            worker.shutdownNow();
+        }
+    }
+
+    @Test
+    public void testLatePublishAfterDatabaseEraseIsRejected() {
+        PublishVersionTask task = newPublishTask(BACKEND_ID, TRANSACTION_ID);
+        database.markDropped();
+        env.eraseDatabase(1L, false);
+        Assertions.assertFalse(AgentTaskQueue.addTask(task));
+        Assertions.assertTrue(task.isFinished());
+        Assertions.assertEquals(TStatusCode.ABORTED, task.getErrorCode());
+
+        Mockito.when(catalog.getDbNullable(1L)).thenReturn(null);
+        
Assertions.assertFalse(AgentTaskQueue.addTask(newPublishTask(BACKEND_ID, 
TRANSACTION_ID + 1)));
+        Assertions.assertEquals(0, AgentTaskQueue.getTaskNum());
+    }
+
+    @Test
+    public void testDatabaseEraseReclaimsConcurrentAdmission() throws 
Exception {
+        CountDownLatch metadataRead = new CountDownLatch(1);
+        CountDownLatch resumeAdmission = new CountDownLatch(1);
+        CountDownLatch databaseDropped = new CountDownLatch(1);
+        Mockito.doAnswer(invocation -> {
+            boolean dropped = (boolean) invocation.callRealMethod();
+            metadataRead.countDown();
+            Assertions.assertTrue(resumeAdmission.await(10, TimeUnit.SECONDS));
+            return dropped;
+        }).when(database).isDropped();
+        ExecutorService workers = Executors.newFixedThreadPool(2);
+        PublishVersionTask task = newPublishTask(BACKEND_ID, TRANSACTION_ID);
+        try {
+            Future<Boolean> admission = workers.submit(() -> {
+                try (MockedStatic<Env> threadEnv = mockEnv()) {
+                    return AgentTaskQueue.addTask(task);
+                }
+            });
+            Assertions.assertTrue(metadataRead.await(10, TimeUnit.SECONDS));
+            Future<?> erasure = workers.submit(() -> {
+                try (MockedStatic<Env> threadEnv = mockEnv()) {
+                    database.markDropped();
+                    databaseDropped.countDown();
+                    env.eraseDatabase(1L, false);
+                }
+            });
+            Assertions.assertTrue(databaseDropped.await(10, TimeUnit.SECONDS));
+            resumeAdmission.countDown();
+            Assertions.assertTrue(admission.get(10, TimeUnit.SECONDS));
+            erasure.get(10, TimeUnit.SECONDS);
+            Assertions.assertFalse(AgentTaskQueue.contains(task));
+            Assertions.assertTrue(task.isFinished());
+            Assertions.assertEquals(TStatusCode.ABORTED, task.getErrorCode());
+            Assertions.assertEquals(0, AgentTaskQueue.getTaskNum());
+        } finally {
+            resumeAdmission.countDown();
+            workers.shutdownNow();
+        }
+    }
+
+    @Test
+    public void testDuplicateRepublishDispatchesNewPartitions() {
+        backend.setAlive(true);
+        TPartitionVersionInfo failedPartition = new TPartitionVersionInfo(2L, 
3L, 0L);
+        TPartitionVersionInfo newPartition = new TPartitionVersionInfo(3L, 4L, 
0L);
+        PublishVersionTask task = newPublishTask(BACKEND_ID, TRANSACTION_ID);
+        task.setFinished(true);
+        task.setErrorCode(TStatusCode.RUNTIME_ERROR);
+        task.addErrorTablets(List.of(10L));
+        AgentTaskQueue.addTask(task);
+        SetMultimap<Long, TPartitionVersionInfo> updates = 
HashMultimap.create();
+        updates.put(TRANSACTION_ID, failedPartition);
+        updates.put(TRANSACTION_ID, newPartition);
+        try (MockedStatic<AgentTaskExecutor> executor = 
Mockito.mockStatic(AgentTaskExecutor.class)) {
+            Deencapsulation.invoke(ReportHandler.class, 
"handleRepublishVersionInfo", Map.of(1L, updates), BACKEND_ID);
+            ArgumentCaptor<AgentBatchTask> batch = 
ArgumentCaptor.forClass(AgentBatchTask.class);
+            executor.verify(() -> AgentTaskExecutor.submit(batch.capture()));
+            Assertions.assertEquals(1, batch.getValue().getAllTasks().size());
+            PublishVersionTask dispatched = (PublishVersionTask) 
batch.getValue().getAllTasks().get(0);
+            Assertions.assertEquals(Set.of(failedPartition, newPartition),
+                    Set.copyOf(dispatched.getPartitionVersionInfos()));
+            Assertions.assertSame(task, AgentTaskQueue.getTask(BACKEND_ID, 
TTaskType.PUBLISH_VERSION, TRANSACTION_ID));
+            Assertions.assertEquals(List.of(failedPartition), 
task.getPartitionVersionInfos());
+
+            executor.reset();
+            Deencapsulation.invoke(ReportHandler.class, "taskReport", 
BACKEND_ID, Collections.emptyMap(), 0L);
+            executor.verify(() -> AgentTaskExecutor.submit(batch.capture()));
+            Assertions.assertEquals(List.of(task), 
batch.getValue().getAllTasks());
+        }
+    }
+
+    @Test
+    public void testRemovedBackendPublishIsTrackedWithoutDispatch() throws 
Exception {
+        infoService.dropBackend(BACKEND_ID);
+        TransactionState transaction = newTransaction();
+        transaction.setTransactionStatus(TransactionStatus.COMMITTED);
+        AgentBatchTask batch = new AgentBatchTask();
+        Deencapsulation.invoke(new PublishVersionDaemon(), 
"addPublishVersionTask",
+                Set.of(BACKEND_ID, HEALTHY_BACKEND_ID), TRANSACTION_ID, 
transaction,
+                List.of(new TPartitionVersionInfo(2L, 3L, 0L)), 
Collections.emptyMap(), 0L, batch);
+        PublishVersionTask rejected = 
transaction.getPublishVersionTasks().get(BACKEND_ID).get(0);
+        Assertions.assertTrue(rejected.isFinished());
+        Assertions.assertEquals(TStatusCode.ABORTED, rejected.getErrorCode());
+        Assertions.assertTrue(rejected.getSuccTablets().isEmpty());
+        Assertions.assertFalse(AgentTaskQueue.contains(rejected));
+        PublishVersionTask accepted = 
transaction.getPublishVersionTasks().get(HEALTHY_BACKEND_ID).get(0);
+        Assertions.assertEquals(List.of(accepted), batch.getAllTasks());
+        Assertions.assertTrue(AgentTaskQueue.contains(accepted));
+    }
+
+    private TransactionState newTransaction() {
+        return new TransactionState(1L, List.of(), TRANSACTION_ID, 
"publish_test", null,
+                TransactionState.LoadJobSourceType.FRONTEND,
+                new 
TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, 0, 
"127.0.0.1", 0L),
+                -1L, 10000L);
+    }
+
+    private MockedStatic<Env> mockEnv() {
+        MockedStatic<Env> result = Mockito.mockStatic(Env.class);
+        result.when(Env::getCurrentEnv).thenReturn(env);
+        result.when(Env::getCurrentSystemInfo).thenReturn(infoService);
+        result.when(Env::getCurrentInternalCatalog).thenReturn(catalog);
+        
result.when(Env::getCurrentGlobalTransactionMgr).thenReturn(transactionMgr);
+        return result;
+    }
+
+    private Map<Long, SetMultimap<Long, TPartitionVersionInfo>> 
republishInfo() {
+        SetMultimap<Long, TPartitionVersionInfo> transactions = 
HashMultimap.create();
+        transactions.put(TRANSACTION_ID, new TPartitionVersionInfo(2L, 3L, 
0L));
+        return Map.of(1L, transactions);
+    }
+
+    private PublishVersionTask newPublishTask(long backendId, long 
transactionId) {
+        return new PublishVersionTask(backendId, transactionId, 1L,
+                List.of(new TPartitionVersionInfo(2L, 3L, 0L)), 0L);
+    }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to