This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.1 by this push:
     new c49e88bf829 branch-4.1: pick some group commit prs(#60652)(#61555) 
(#61953)
c49e88bf829 is described below

commit c49e88bf829935cd6b4faed9f44703f4a027cf75
Author: meiyi <[email protected]>
AuthorDate: Tue Mar 31 22:06:25 2026 +0800

    branch-4.1: pick some group commit prs(#60652)(#61555) (#61953)
    
    pick
    1. Fix NPE in group commit when backend belongs to a different cluster
    (https://github.com/apache/doris/pull/60652)
    2. group commit select backend should consider isLoadAvailable
    (https://github.com/apache/doris/pull/61555)
    
    ---------
    
    Co-authored-by: deardeng <[email protected]>
---
 .../org/apache/doris/load/GroupCommitManager.java  | 30 ++++++++++++++++++----
 .../main/java/org/apache/doris/qe/Coordinator.java | 26 +++++++++++++++++++
 2 files changed, 51 insertions(+), 5 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java 
b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
index 25e72199d2a..bea5cd03c7d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
@@ -213,7 +213,15 @@ public class GroupCommitManager {
             try {
                 long backendId = new MasterOpExecutor(context)
                         .getGroupCommitLoadBeId(tableId, clusterName);
-                return Env.getCurrentSystemInfo().getBackend(backendId);
+                Backend be = Env.getCurrentSystemInfo().getBackend(backendId);
+                if (LOG.isDebugEnabled()) {
+                    LOG.debug("selectBackendForGroupCommit on non-master: 
tableId={}, clusterName={},"
+                            + " backendId={}, backend={}, backendCluster={}",
+                            tableId, clusterName, backendId,
+                            be != null ? be.getHost() + ":" + be.getBePort() : 
"null",
+                            be != null ? be.getCloudClusterName() : "null");
+                }
+                return be;
             } catch (Exception e) {
                 throw new LoadException(e.getMessage());
             }
@@ -349,8 +357,7 @@ public class GroupCommitManager {
                     return null;
                 }
                 Backend backend = 
Env.getCurrentSystemInfo().getBackend(backendId);
-                if (backend != null && backend.isAlive() && 
!backend.isDecommissioned()
-                        && (!Config.isCloudMode() || 
!backend.isDecommissioning())) {
+                if (isBackendAvailable(backend, cluster)) {
                     return backend.getId();
                 } else {
                     tableToBeMap.remove(encode(cluster, tableId));
@@ -362,13 +369,26 @@ public class GroupCommitManager {
         return null;
     }
 
+    private boolean isBackendAvailable(Backend backend, String cluster) {
+        if (backend == null || !backend.isAlive() || 
backend.isDecommissioned() || !backend.isLoadAvailable()) {
+            return false;
+        }
+        if (!Config.isCloudMode()) {
+            return true;
+        }
+        // for cloud mode
+        if (backend.isDecommissioning()) {
+            return false;
+        }
+        return cluster == null || 
cluster.equals(backend.getCloudClusterName());
+    }
+
     @Nullable
     private Long getRandomBackend(String cluster, long tableId, List<Backend> 
backends) {
         OlapTable table = (OlapTable) 
Env.getCurrentEnv().getInternalCatalog().getTableByTableId(tableId);
         Collections.shuffle(backends);
         for (Backend backend : backends) {
-            if (backend.isAlive() && !backend.isDecommissioned() && 
(!Config.isCloudMode()
-                    || !backend.isDecommissioning())) {
+            if (isBackendAvailable(backend, cluster)) {
                 tableToBeMap.put(encode(cluster, tableId), backend.getId());
                 tableToPressureMap.put(tableId,
                         new 
SlidingWindowCounter(table.getGroupCommitIntervalMs() / 1000 + 1));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java 
b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index c2227b280b8..397ddfb982b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -567,6 +567,26 @@ public class Coordinator implements CoordInterface {
 
         this.idToBackend = 
Env.getCurrentSystemInfo().getBackendsByCurrentCluster();
 
+        // Log cluster info and groupCommitBackend for debugging NPE in 
PipelineExecContext
+        if (groupCommitBackend != null) {
+            String currentCluster = "unknown";
+            try {
+                if (ConnectContext.get() != null) {
+                    currentCluster = ConnectContext.get().getCloudCluster();
+                }
+            } catch (Exception e) {
+                LOG.debug("failed to get current cloud cluster for debug log", 
e);
+            }
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("query {} prepare: currentCluster={}, 
idToBackend.size={}, idToBackend.keys={},"
+                                + " groupCommitBackend=[id={}, host={}, 
cluster={}], groupCommitBackendInMap={}",
+                        DebugUtil.printId(queryId), currentCluster, 
idToBackend.size(), idToBackend.keySet(),
+                        groupCommitBackend.getId(), 
groupCommitBackend.getHost(),
+                        groupCommitBackend.getCloudClusterName(),
+                        idToBackend.containsKey(groupCommitBackend.getId()));
+            }
+        }
+
         if (LOG.isDebugEnabled()) {
             int backendNum = idToBackend.size();
             StringBuilder backendInfos = new StringBuilder("backends info:");
@@ -838,6 +858,12 @@ public class Coordinator implements CoordInterface {
                 // So that we can use one RPC to send all fragment instances 
of a BE.
                 for (Map.Entry<TNetworkAddress, TPipelineFragmentParams> entry 
: tParams.entrySet()) {
                     Long backendId = 
this.addressToBackendID.get(entry.getKey());
+                    if (backendId == null) {
+                        LOG.warn("query {} sendPipelineCtx: addressToBackendID 
lookup returned null!"
+                                + " address={}, fragmentId={}, 
addressToBackendID={}",
+                                DebugUtil.printId(queryId), entry.getKey(),
+                                fragment.getFragmentId(), addressToBackendID);
+                    }
                     backendFragments.add(Pair.of(fragment.getFragmentId(), 
backendId));
                     PipelineExecContext pipelineExecContext = new 
PipelineExecContext(fragment.getFragmentId(),
                             entry.getValue(), idToBackend.get(backendId), 
executionProfile, jobId);


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to