This is an automated email from the ASF dual-hosted git repository.
dgovorukhin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new bfcb0c3 IGNITE-12351 Append additional cp tracking activity - pages
sort - Fixes #7036.
bfcb0c3 is described below
commit bfcb0c38fa71c713651180459e674624993a6fe5
Author: vmalin <vmalinovs...@gridgain.com>
AuthorDate: Tue Nov 12 15:15:12 2019 +0300
IGNITE-12351 Append additional cp tracking activity - pages sort - Fixes
#7036.
Signed-off-by: Dmitriy Govorukhin <dmitriy.govoruk...@gmail.com>
---
.../apache/ignite/internal/pagemem/FullPageId.java | 7 ++
.../GridCacheDatabaseSharedManager.java | 25 +++--
.../pagemem/CheckpointMetricsTracker.java | 76 ++++++++++------
.../cache/persistence/pagemem/PageMemoryImpl.java | 10 +-
.../db/CheckpointBufferDeadlockTest.java | 4 +-
.../db/checkpoint/CheckpointStartLoggingTest.java | 101 +++++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite2.java | 3 +
7 files changed, 183 insertions(+), 43 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
index 2d9aa4d..4c7d031 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
@@ -125,6 +125,13 @@ public class FullPageId {
}
/**
+ * @return Effective page ID.
+ */
+ public long effectivePageId() {
+ return effectivePageId;
+ }
+
+ /**
* @return Cache group ID.
*/
public int groupId() {
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 5831f4e..aaf4a44 100755
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -96,7 +96,6 @@ import org.apache.ignite.internal.mem.DirectMemoryRegion;
import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
@@ -3510,9 +3509,18 @@ public class GridCacheDatabaseSharedManager extends
IgniteCacheDatabaseSharedMan
@SuppressWarnings("NakedNotify")
public class Checkpointer extends GridWorker {
/** Checkpoint started log message format. */
- private static final String CHECKPOINT_STARTED_LOG_FORMAT =
"Checkpoint started [checkpointId=%s, startPtr=%s," +
- " checkpointBeforeLockTime=%dms, checkpointLockWait=%dms,
checkpointListenersExecuteTime=%dms, " +
- "checkpointLockHoldTime=%dms, walCpRecordFsyncDuration=%dms, %s
pages=%d, reason='%s']";
+ private static final String CHECKPOINT_STARTED_LOG_FORMAT =
"Checkpoint started [" +
+ "checkpointId=%s, " +
+ "startPtr=%s, " +
+ "checkpointBeforeLockTime=%dms, " +
+ "checkpointLockWait=%dms, " +
+ "checkpointListenersExecuteTime=%dms, " +
+ "checkpointLockHoldTime=%dms, " +
+ "walCpRecordFsyncDuration=%dms, " +
+ "writeCheckpointEntryDuration=%dms, " +
+ "splitAndSortCpPagesDuration=%dms, " +
+ "%s pages=%d, " +
+ "reason='%s']";
/** Temporary write buffer. */
private final ByteBuffer tmpWriteBuf;
@@ -4185,9 +4193,13 @@ public class GridCacheDatabaseSharedManager extends
IgniteCacheDatabaseSharedMan
curr.cpMarkerStored.onDone();
+ tracker.onSplitAndSortCpPagesStart();
+
GridMultiCollectionWrapper<FullPageId> cpPages =
splitAndSortCpPagesIfNeeded(
cpPagesTuple, persistenceCfg.getCheckpointThreads());
+ tracker.onSplitAndSortCpPagesEnd();
+
if (printCheckpointStats && log.isInfoEnabled()) {
long possibleJvmPauseDur =
possibleLongJvmPauseDuration(tracker);
@@ -4201,6 +4213,8 @@ public class GridCacheDatabaseSharedManager extends
IgniteCacheDatabaseSharedMan
tracker.listenersExecuteDuration(),
tracker.lockHoldDuration(),
tracker.walCpRecordFsyncDuration(),
+ tracker.writeCheckpointEntryDuration(),
+ tracker.splitAndSortCpPagesDuration(),
possibleJvmPauseDur > 0 ?
"possibleJvmPauseDuration=" + possibleJvmPauseDur + "ms," : "",
cpPages.size(),
curr.reason
@@ -4640,8 +4654,7 @@ public class GridCacheDatabaseSharedManager extends
IgniteCacheDatabaseSharedMan
if (cmp != 0)
return cmp;
- return
Long.compare(PageIdUtils.effectivePageId(o1.pageId()),
- PageIdUtils.effectivePageId(o2.pageId()));
+ return Long.compare(o1.effectivePageId(),
o2.effectivePageId());
}
};
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
index d0ffefd..55fb19a 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
@@ -17,8 +17,12 @@
package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import
org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import
org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry;
+import
org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntryType;
/**
* Tracks various checkpoint phases and stats.
@@ -77,6 +81,12 @@ public class CheckpointMetricsTracker {
private long walCpRecordFsyncEnd;
/** */
+ private long splitAndSortCpPagesStart;
+
+ /** */
+ private long splitAndSortCpPagesEnd;
+
+ /** */
private long listenersExecEnd;
/**
@@ -86,9 +96,7 @@ public class CheckpointMetricsTracker {
COW_PAGES_UPDATER.incrementAndGet(this);
}
- /**
- *
- */
+ /** */
public void onDataPageWritten() {
DATA_PAGES_UPDATER.incrementAndGet(this);
}
@@ -107,65 +115,57 @@ public class CheckpointMetricsTracker {
return dataPages;
}
- /**
- *
- */
+ /** */
public void onLockWaitStart() {
cpLockWaitStart = System.currentTimeMillis();
}
- /**
- *
- */
+ /** */
public void onMarkStart() {
cpMarkStart = System.currentTimeMillis();
}
- /**
- *
- */
+ /** */
public void onLockRelease() {
cpLockRelease = System.currentTimeMillis();
}
- /**
- *
- */
+ /** */
public void onPagesWriteStart() {
cpPagesWriteStart = System.currentTimeMillis();
}
- /**
- *
- */
+ /** */
public void onFsyncStart() {
cpFsyncStart = System.currentTimeMillis();
}
- /**
- *
- */
+ /** */
public void onEnd() {
cpEnd = System.currentTimeMillis();
}
- /**
- *
- */
+ /** */
public void onListenersExecuteEnd() {
listenersExecEnd = System.currentTimeMillis();
}
- /**
- *
- */
+ /** */
public void onWalCpRecordFsyncStart() {
walCpRecordFsyncStart = System.currentTimeMillis();
}
- /**
- *
- */
+ /** */
+ public void onSplitAndSortCpPagesStart() {
+ splitAndSortCpPagesStart = System.currentTimeMillis();
+ }
+
+ /** */
+ public void onSplitAndSortCpPagesEnd() {
+ splitAndSortCpPagesEnd = System.currentTimeMillis();
+ }
+
+ /** */
public void onWalCpRecordFsyncEnd() {
walCpRecordFsyncEnd = System.currentTimeMillis();
}
@@ -234,6 +234,22 @@ public class CheckpointMetricsTracker {
}
/**
+ * @return Duration of checkpoint entry buffer writing to file.
+ *
+ * @see GridCacheDatabaseSharedManager#writeCheckpointEntry(ByteBuffer,
CheckpointEntry, CheckpointEntryType)
+ */
+ public long writeCheckpointEntryDuration() {
+ return splitAndSortCpPagesStart - walCpRecordFsyncEnd;
+ }
+
+ /**
+ * @return Duration of splitting and sorting checkpoint pages.
+ */
+ public long splitAndSortCpPagesDuration() {
+ return splitAndSortCpPagesEnd - splitAndSortCpPagesStart;
+ }
+
+ /**
* @return Checkpoint start time.
*/
public long checkpointStartTime() {
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index fcc6efe..964d618 100755
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1234,7 +1234,7 @@ public class PageMemoryImpl implements PageMemoryEx {
relPtr = refreshOutdatedPage(
seg,
fullId.groupId(),
- PageIdUtils.effectivePageId(fullId.pageId()),
+ fullId.effectivePageId(),
true
);
@@ -1394,7 +1394,7 @@ public class PageMemoryImpl implements PageMemoryEx {
private long resolveRelativePointer(Segment seg, FullPageId fullId, int
reqVer) {
return seg.loadedPages.get(
fullId.groupId(),
- PageIdUtils.effectivePageId(fullId.pageId()),
+ fullId.effectivePageId(),
reqVer,
INVALID_REL_PTR,
OUTDATED_REL_PTR
@@ -1524,7 +1524,7 @@ public class PageMemoryImpl implements PageMemoryEx {
*/
public boolean hasLoadedPage(FullPageId fullPageId) {
int grpId = fullPageId.groupId();
- long pageId = PageIdUtils.effectivePageId(fullPageId.pageId());
+ long pageId = fullPageId.effectivePageId();
int partId = PageIdUtils.partId(pageId);
Segment seg = segment(grpId, pageId);
@@ -2481,7 +2481,7 @@ public class PageMemoryImpl implements PageMemoryEx {
loadedPages.remove(
fullPageId.groupId(),
- PageIdUtils.effectivePageId(fullPageId.pageId())
+ fullPageId.effectivePageId()
);
return relRmvAddr;
@@ -2554,7 +2554,7 @@ public class PageMemoryImpl implements PageMemoryEx {
if (preparePageRemoval(fullPageId, absEvictAddr,
saveDirtyPage)) {
loadedPages.remove(
fullPageId.groupId(),
- PageIdUtils.effectivePageId(fullPageId.pageId())
+ fullPageId.effectivePageId()
);
return addr;
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
index 28acd17..e96a84b 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
@@ -234,11 +234,11 @@ public class CheckpointBufferDeadlockTest extends
GridCommonAbstractTest {
pickedPages.sort(new Comparator<FullPageId>() {
@Override public int compare(FullPageId o1,
FullPageId o2) {
int cmp = Long.compare(o1.groupId(),
o2.groupId());
+
if (cmp != 0)
return cmp;
- return
Long.compare(PageIdUtils.effectivePageId(o1.pageId()),
-
PageIdUtils.effectivePageId(o2.pageId()));
+ return Long.compare(o1.effectivePageId(),
o2.effectivePageId());
}
});
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/CheckpointStartLoggingTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/CheckpointStartLoggingTest.java
new file mode 100644
index 0000000..1fc0885
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/CheckpointStartLoggingTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.checkpoint;
+
+import java.util.regex.Pattern;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.ListeningTestLogger;
+import org.apache.ignite.testframework.LogListener;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * Checks correct checkpoint times logging.
+ */
+public class CheckpointStartLoggingTest extends GridCommonAbstractTest {
+ /** */
+ private static final String VALID_MS_PATTERN = "[0-9]*ms";
+
+ /** */
+ private static final String CHECKPOINT_STARTED_LOG_FORMAT = "Checkpoint
started .*" +
+ "checkpointBeforeLockTime=" + VALID_MS_PATTERN + ", " +
+ "checkpointLockWait=" + VALID_MS_PATTERN + ", " +
+ "checkpointListenersExecuteTime="+ VALID_MS_PATTERN +", " +
+ "checkpointLockHoldTime="+ VALID_MS_PATTERN + ", " +
+ "walCpRecordFsyncDuration="+ VALID_MS_PATTERN +", " +
+ "writeCheckpointEntryDuration="+ VALID_MS_PATTERN +", " +
+ "splitAndSortCpPagesDuration="+ VALID_MS_PATTERN +", " +
+ ".* pages=[1-9][0-9]*, " +
+ "reason=.*";
+
+ /** */
+ private ListeningTestLogger testLogger = new ListeningTestLogger(false,
log);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ DataStorageConfiguration memCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(10 * 1024 * 1024)
+ .setPersistenceEnabled(true));
+
+ cfg.setDataStorageConfiguration(memCfg);
+
+ cfg.setGridLogger(testLogger);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ super.afterTest();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ @Test
+ public void testCheckpointLogging() throws Exception {
+ LogListener lsnr =
LogListener.matches(Pattern.compile(CHECKPOINT_STARTED_LOG_FORMAT)).build();
+
+ testLogger.registerListener(lsnr);
+
+ Ignite ignite = startGrid();
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Integer, Integer> cache =
ignite.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ for (int i = 0; i < 1000; i++)
+ cache.put(i, i);
+
+ forceCheckpoint();
+
+ assertTrue(lsnr.check());
+ }
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 3cdfff2..8a71611 100644
---
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -45,6 +45,7 @@ import
org.apache.ignite.internal.processors.cache.persistence.db.IgniteShutdown
import
org.apache.ignite.internal.processors.cache.persistence.db.SlowHistoricalRebalanceSmallHistoryTest;
import
org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.CheckpointFailBeforeWriteMarkTest;
import
org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.CheckpointFreeListTest;
+import
org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.CheckpointStartLoggingTest;
import
org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
import
org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
import
org.apache.ignite.internal.processors.cache.persistence.db.wal.FsyncWalRolloverDoesNotBlockTest;
@@ -216,6 +217,8 @@ public class IgnitePdsTestSuite2 {
GridTestUtils.addTestIfNeeded(suite, CheckpointFreeListTest.class,
ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, CheckpointStartLoggingTest.class,
ignoredTests);
+
GridTestUtils.addTestIfNeeded(suite, FreeListCachingTest.class,
ignoredTests);
GridTestUtils.addTestIfNeeded(suite,
IgniteWalIteratorSwitchSegmentTest.class, ignoredTests);
