This is an automated email from the ASF dual-hosted git repository.

NSAmelchev pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git


The following commit(s) were added to refs/heads/master by this push:
     new b70599322ee IGNITE-29159 Add explicit lock hold time to system view 
and diagnostic dumps (#13700)
b70599322ee is described below

commit b70599322eefcf66ae325ee726c8ae1c1c0b6fd0
Author: Nikita Amelchev <[email protected]>
AuthorDate: Thu Oct 22 21:07:01 2026 +0300

    IGNITE-29159 Add explicit lock hold time to system view and diagnostic 
dumps (#13700)
---
 docs/_docs/monitoring-metrics/system-views.adoc    |   1 +
 .../ignite/jdbc/thin/JdbcThinMetadataSelfTest.java |   1 +
 .../cache/GridCacheExplicitLockSpan.java           |  33 +++--
 .../processors/cache/GridCacheMvccManager.java     |   2 +-
 .../cache/GridCachePartitionExchangeManager.java   |  33 ++++-
 .../transactions/TransactionMetricsAdapter.java    |   4 +-
 .../apache/ignite/internal/util/IgniteUtils.java   |   5 +-
 .../spi/systemview/view/CacheExplicitLockView.java |  16 ++-
 .../internal/metric/SystemViewLocksTest.java       |   4 +
 .../distributed/dht/ExplicitLockDumpTest.java      | 142 +++++++++++++++++++++
 .../ignite/testsuites/IgniteCacheTestSuite14.java  |   2 +
 11 files changed, 227 insertions(+), 16 deletions(-)

diff --git a/docs/_docs/monitoring-metrics/system-views.adoc 
b/docs/_docs/monitoring-metrics/system-views.adoc
index 969b9f20619..ea9cd2117ab 100644
--- a/docs/_docs/monitoring-metrics/system-views.adoc
+++ b/docs/_docs/monitoring-metrics/system-views.adoc
@@ -314,6 +314,7 @@ Explicit locks are acquired using the `Cache.lock(key)` API.
 |KEY | string | Locked key
 |THREAD_ID | long | ID of the thread that acquired the lock
 |XID | UUID | Lock ID (unique identifier)
+|DURATION | long | Hold duration in milliseconds, 0 until the lock is acquired
 |===
 
 == CACHE_LOCKS
diff --git 
a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java
 
b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java
index 13e5438399a..d4dd8d94005 100644
--- 
a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java
+++ 
b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMetadataSelfTest.java
@@ -1187,6 +1187,7 @@ public class JdbcThinMetadataSelfTest extends 
JdbcThinAbstractSelfTest {
                 "SYS.CACHE_EXPLICIT_LOCKS.KEY.null",
                 "SYS.CACHE_EXPLICIT_LOCKS.THREAD_ID.null",
                 "SYS.CACHE_EXPLICIT_LOCKS.XID.null",
+                "SYS.CACHE_EXPLICIT_LOCKS.DURATION.null",
                 "SYS.CACHE_LOCKS.CACHE_ID.null",
                 "SYS.CACHE_LOCKS.KEY.null",
                 "SYS.CACHE_LOCKS.NODE_ID.null",
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java
index fb7025a9425..8c09abcd5c9 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheExplicitLockSpan.java
@@ -290,20 +290,33 @@ public class GridCacheExplicitLockSpan extends 
ReentrantLock {
         }
     }
 
+    /** @return How long the key has been locked for, in milliseconds, or 0 if 
the lock is not acquired yet. */
+    public long holdTime(IgniteTxKey key) {
+        lock();
+
+        try {
+            KeyCandidates keyCands = cands.get(key);
+
+            return keyCands == null ? 0 : 
keyCands.holdTime(U.currentTimeMillis());
+        }
+        finally {
+            unlock();
+        }
+    }
+
     /**
-     * @param now Current time.
      * @return Longest time a key of the span has been locked for, in 
milliseconds, or 0 if no lock is acquired yet.
      */
-    public long maxHoldTime(long now) {
+    public long maxHoldTime() {
         lock();
 
         try {
+            long now = U.currentTimeMillis();
+
             long max = 0;
 
-            for (KeyCandidates keyCands : cands.values()) {
-                if (keyCands.acquireTime > 0)
-                    max = Math.max(max, now - keyCands.acquireTime);
-            }
+            for (KeyCandidates keyCands : cands.values())
+                max = Math.max(max, keyCands.holdTime(now));
 
             return max;
         }
@@ -408,10 +421,14 @@ public class GridCacheExplicitLockSpan extends 
ReentrantLock {
                 acquireTime = U.currentTimeMillis();
         }
 
+        /** */
+        long holdTime(long now) {
+            return acquireTime == 0 ? 0 : now - acquireTime;
+        }
+
         /** {@inheritDoc} */
         @Override public String toString() {
-            return S.toString(KeyCandidates.class, this,
-                "holdTime", acquireTime == 0 ? 0 : U.currentTimeMillis() - 
acquireTime, false);
+            return S.toString(KeyCandidates.class, this, "holdTime", 
holdTime(U.currentTimeMillis()), false);
         }
     }
 }
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
index 5bfbd0024ae..2a82a6a6ed4 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
@@ -312,7 +312,7 @@ public class GridCacheMvccManager extends 
GridCacheSharedManagerAdapter {
             new CacheExplicitLockViewWalker(),
             pendingExplicit.values(),
             GridCacheExplicitLockSpan::candidates,
-            (threadId, lock) -> new CacheExplicitLockView(lock)
+            CacheExplicitLockView::new
         );
 
         List<GridCacheMvccCandidate> locks = new ArrayList<>();
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 9dcc18d5e05..c63df8c543b 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2049,6 +2049,22 @@ public class GridCachePartitionExchangeManager<K, V> 
extends GridCacheSharedMana
         }
 
         if (mvcc != null) {
+            WarningsGroup lockWarnings = new WarningsGroup("First %d long 
running explicit locks [total=%d]",
+                diagnosticLog, DIAGNOSTIC_WARN_LIMIT);
+
+            for (GridCacheExplicitLockSpan lockSpan : 
mvcc.activeExplicitLocks()) {
+                if (lockSpan.maxHoldTime() > timeout) {
+                    found = true;
+
+                    if (lockWarnings.canAddMessage())
+                        lockWarnings.add(explicitLockWarning(lockSpan));
+                    else
+                        lockWarnings.incTotal();
+                }
+            }
+
+            lockWarnings.printToLog();
+
             WarningsGroup activeWarnings = new WarningsGroup("First %d long 
running cache futures [total=%d]",
                 diagnosticLog, DIAGNOSTIC_WARN_LIMIT);
 
@@ -2249,6 +2265,21 @@ public class GridCachePartitionExchangeManager<K, V> 
extends GridCacheSharedMana
             task instanceof ForceRebalanceExchangeTask;
     }
 
+    /** @return Span and, if it holds a lock, the stack trace of the holding 
thread. */
+    private static String explicitLockWarning(GridCacheExplicitLockSpan 
lockSpan) {
+        GridStringBuilder sb = new GridStringBuilder(">>> ").a(lockSpan);
+
+        GridCacheMvccCandidate cand = F.first(lockSpan.candidates());
+
+        if (cand != null && lockSpan.maxHoldTime() > 0) {
+            sb.a(U.nl()).a("Stack trace of the explicit lock owner 
thread:").a(U.nl());
+
+            U.printStackTrace(cand.threadId(), sb);
+        }
+
+        return sb.toString();
+    }
+
     /**
      * @param exchTopVer Exchange topology version.
      * @param diagCtx Diagnostic request.
@@ -2289,7 +2320,7 @@ public class GridCachePartitionExchangeManager<K, V> 
extends GridCacheSharedMana
                     first = false;
                 }
 
-                U.warn(diagnosticLog, ">>> " + lockSpan);
+                U.warn(diagnosticLog, explicitLockWarning(lockSpan));
             }
 
             first = true;
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
index 00640b20830..e511b4a8bee 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
@@ -440,12 +440,10 @@ public class TransactionMetricsAdapter implements 
TransactionMetrics {
         if (mvccMgr == null)
             return 0;
 
-        long now = U.currentTimeMillis();
-
         long max = 0;
 
         for (GridCacheExplicitLockSpan span : mvccMgr.activeExplicitLocks())
-            max = Math.max(max, span.maxHoldTime(now));
+            max = Math.max(max, span.maxHoldTime());
 
         return max;
     }
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java 
b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index f0c4b094aa9..cbcd98495d0 100755
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -937,7 +937,10 @@ public abstract class IgniteUtils extends CommonUtils {
 
         ThreadInfo threadInfo = mxBean.getThreadInfo(threadId, 
Integer.MAX_VALUE);
 
-        printThreadInfo(threadInfo, sb, Collections.<Long>emptySet());
+        if (threadInfo == null)
+            sb.a("Thread [id=").a(threadId).a("] is not alive or is a virtual 
thread").a(NL);
+        else
+            printThreadInfo(threadInfo, sb, Collections.<Long>emptySet());
     }
 
     /**
diff --git 
a/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/CacheExplicitLockView.java
 
b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/CacheExplicitLockView.java
index 8d2a36d6703..38a705f8e8c 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/CacheExplicitLockView.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/spi/systemview/view/CacheExplicitLockView.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.spi.systemview.view;
 
 import org.apache.ignite.internal.binary.BinaryUtils;
+import org.apache.ignite.internal.processors.cache.GridCacheExplicitLockSpan;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.systemview.Order;
 import org.apache.ignite.internal.systemview.SystemViewDescriptor;
@@ -29,13 +30,18 @@ import org.apache.ignite.lang.IgniteUuid;
  */
 @SystemViewDescriptor
 public class CacheExplicitLockView {
+    /** */
+    private final GridCacheExplicitLockSpan span;
+
     /** */
     private final GridCacheMvccCandidate cand;
 
     /**
-     * @param cand Lock candidate
+     * @param span Lock span of the candidate.
+     * @param cand Lock candidate.
      */
-    public CacheExplicitLockView(GridCacheMvccCandidate cand) {
+    public CacheExplicitLockView(GridCacheExplicitLockSpan span, 
GridCacheMvccCandidate cand) {
+        this.span = span;
         this.cand = cand;
     }
 
@@ -70,4 +76,10 @@ public class CacheExplicitLockView {
     public IgniteUuid xid() {
         return BinaryUtils.asIgniteUuid(cand.version());
     }
+
+    /** @return Hold duration in milliseconds, 0 until the lock is acquired. */
+    @Order(4)
+    public long duration() {
+        return span.holdTime(cand.key());
+    }
 }
diff --git 
a/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewLocksTest.java
 
b/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewLocksTest.java
index e7b6758819c..e61995de0fc 100644
--- 
a/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewLocksTest.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/internal/metric/SystemViewLocksTest.java
@@ -119,6 +119,10 @@ public class SystemViewLocksTest extends 
SystemViewAbstractTest {
             assertEquals(explicitLocks1.get(0).threadId(), 
explicitLocks1.get(1).threadId());
             assertNotSame(explicitLocks2.get(0).threadId(), 
explicitLocks2.get(1).threadId());
 
+            // Hold duration is reported per key for acquired locks only: 
node1 holds key2 and waits for key0.
+            assertTrue(waitForCondition(() -> 
explicitLocks0.stream().allMatch(l -> l.duration() > 0), 1_000L));
+            assertTrue(waitForCondition(() -> F.size(explicitLocks1, l -> 
l.duration() > 0) == 1, 1_000L));
+
             // Check lock owners.
             CacheLockView owner0 = F.find(keyLocks0, null, 
CacheLockView::isOwner);
             CacheLockView owner1 = F.find(keyLocks1, null, 
CacheLockView::isOwner);
diff --git 
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockDumpTest.java
 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockDumpTest.java
new file mode 100644
index 00000000000..5e78a712f74
--- /dev/null
+++ 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/ExplicitLockDumpTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.concurrent.locks.Lock;
+import java.util.regex.Pattern;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.testframework.ListeningTestLogger;
+import org.apache.ignite.testframework.LogListener;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static 
org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/**
+ * Tests that the diagnostics of a long held explicit lock show how long the 
lock has been held and what the thread
+ * holding it is doing: in the long running operations dump and in the dump of 
an exchange blocked by the lock.
+ */
+public class ExplicitLockDumpTest extends GridCommonAbstractTest {
+    /** */
+    private static final String OWNER_DUMP = "(?s)>>> 
GridCacheExplicitLockSpan \\[.*holdTime=[1-9].*" +
+        "Stack trace of the explicit lock owner 
thread:.*ExplicitLockDumpTest\\.";
+
+    /** */
+    private final ListeningTestLogger listeningLog = new 
ListeningTestLogger(log);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String 
igniteInstanceName) throws Exception {
+        return super.getConfiguration(igniteInstanceName)
+            .setGridLogger(listeningLog)
+            // 2 * networkTimeout is the pending objects dump timeout of the 
exchange, see
+            // GridDhtPartitionsExchangeFuture#waitPartitionRelease and 
ExchangeWorker#body.
+            .setNetworkTimeout(1000)
+            .setCacheConfiguration(new 
CacheConfiguration<>(DEFAULT_CACHE_NAME).setAtomicityMode(TRANSACTIONAL));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /** The lock held longer than the long operations dump timeout is dumped 
without an exchange. */
+    @Test
+    @WithSystemProperty(key = IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, value = 
"500")
+    public void testLongRunningDump() throws Exception {
+        Lock lock = startGrid(0).cache(DEFAULT_CACHE_NAME).lock(0);
+
+        LogListener lsnr = LogListener.matches(Pattern.compile("First \\d+ 
long running explicit locks \\[total=1\\]"))
+            .andMatches(Pattern.compile(OWNER_DUMP + 
"testLongRunningDump\\(")).build();
+
+        listeningLog.registerListener(lsnr);
+
+        lock.lock();
+
+        try {
+            assertTrue(waitForCondition(lsnr::check, getTestTimeout()));
+        }
+        finally {
+            lock.unlock();
+        }
+    }
+
+    /** The server waits for its own lock on partition release. */
+    @Test
+    public void testServerLockOwnerDump() throws Exception {
+        checkOwnerDump(startGrid(0));
+    }
+
+    /** The client does not wait for partition release, its exchange worker 
dumps the lock when the exchange hangs. */
+    @Test
+    public void testClientLockOwnerDump() throws Exception {
+        startGrid(0);
+
+        checkOwnerDump(startClientGrid(1));
+    }
+
+    /** The lock is left by a terminated thread, so there is no stack trace to 
show. */
+    @Test
+    public void testLeakedLockDump() throws Exception {
+        IgniteEx ignite = startGrid(0);
+
+        runAsync(() -> ignite.cache(DEFAULT_CACHE_NAME).lock(0).lock()).get();
+
+        checkExchangeDump("Thread \\[id=\\d+\\] is not alive");
+    }
+
+    /** @param lockNode Node that acquires the lock in the test thread. */
+    private void checkOwnerDump(IgniteEx lockNode) throws Exception {
+        Lock lock = lockNode.cache(DEFAULT_CACHE_NAME).lock(0);
+
+        lock.lock();
+
+        IgniteInternalFuture<?> fut;
+
+        try {
+            fut = checkExchangeDump(OWNER_DUMP + "checkOwnerDump\\(");
+        }
+        finally {
+            lock.unlock();
+        }
+
+        fut.get(getTestTimeout());
+    }
+
+    /** Starts a cache, which is blocked by the lock, and waits for the 
exchange diagnostics. */
+    private IgniteInternalFuture<?> checkExchangeDump(String regex) throws 
Exception {
+        LogListener lsnr = LogListener.matches(Pattern.compile(regex)).build();
+
+        listeningLog.registerListener(lsnr);
+
+        IgniteInternalFuture<?> fut = runAsync(() -> 
grid(0).createCache("blocked"));
+
+        assertTrue(regex, waitForCondition(lsnr::check, getTestTimeout()));
+        assertFalse(fut.isDone());
+
+        return fut;
+    }
+}
diff --git 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java
 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java
index 21e88dbbe11..76aafeb28bf 100644
--- 
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite14.java
@@ -49,6 +49,7 @@ import 
org.apache.ignite.internal.processors.cache.distributed.IgniteCacheClient
 import 
org.apache.ignite.internal.processors.cache.distributed.IgniteCacheServerNodeConcurrentStart;
 import 
org.apache.ignite.internal.processors.cache.distributed.dht.CachePartitionPartialCountersMapSelfTest;
 import 
org.apache.ignite.internal.processors.cache.distributed.dht.ExplicitLockCancelOnNodeStopTest;
+import 
org.apache.ignite.internal.processors.cache.distributed.dht.ExplicitLockDumpTest;
 import 
org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedDebugTest;
 import 
org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedPreloadRestartSelfTest;
 import 
org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedPrimarySyncSelfTest;
@@ -127,6 +128,7 @@ public class IgniteCacheTestSuite14 {
         GridTestUtils.addTestIfNeeded(suite, 
GridCachePartitionedExplicitLockNodeFailureSelfTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, 
CacheLockReleaseNodeLeaveTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, 
ExplicitLockCancelOnNodeStopTest.class, ignoredTests);
+        GridTestUtils.addTestIfNeeded(suite, ExplicitLockDumpTest.class, 
ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, 
GridCachePartitionedNestedTxTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, 
GridCachePartitionedTxConcurrentGetTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, 
GridCachePartitionedTxReadTest.class, ignoredTests);

Reply via email to