This is an automated email from the ASF dual-hosted git repository.
agoncharuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new dc4b71b IGNITE-13786 Add defragmentation-specific B+Tree
optimizations - Fixes #8560.
dc4b71b is described below
commit dc4b71b230e289e5b0d99b00e0632c9d1d0e6e82
Author: ibessonov <bessonov...@gmail.com>
AuthorDate: Thu Dec 10 18:05:04 2020 +0300
IGNITE-13786 Add defragmentation-specific B+Tree optimizations - Fixes
#8560.
Signed-off-by: Alexey Goncharuk <alexey.goncha...@gmail.com>
---
.../CachePartitionDefragmentationManager.java | 24 ++++++++++++++++++----
.../cache/persistence/tree/BPlusTree.java | 23 +++++++++++++++++----
.../cache/persistence/tree/util/InsertLast.java | 24 ----------------------
.../persistence/IgnitePdsDefragmentationTest.java | 2 +-
.../defragmentation/IndexingDefragmentation.java | 6 ++++--
5 files changed, 44 insertions(+), 35 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java
index 75b3458..aa82a23 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java
@@ -93,6 +93,7 @@ import static
org.apache.ignite.internal.processors.cache.persistence.Checkpoint
import static
org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.DEFRAGMENTATION_MAPPING_REGION_NAME;
import static
org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.DEFRAGMENTATION_PART_REGION_NAME;
import static
org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.batchRenameDefragmentedCacheGroupPartitions;
+import static
org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedIndexFile;
import static
org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedIndexTmpFile;
import static
org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartFile;
import static
org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartMappingFile;
@@ -386,9 +387,9 @@ public class CachePartitionDefragmentationManager {
partCtx.partPageMemory
);
- partCtx.createNewCacheDataStore(offheap);
+ partCtx.createNewCacheDataStore(offheap);
- copyPartitionData(partCtx, treeIter);
+ copyPartitionData(partCtx, treeIter);
DefragmentationPageReadWriteManager pageMgr =
(DefragmentationPageReadWriteManager)partCtx.partPageMemory.pageManager();
@@ -450,7 +451,21 @@ public class CachePartitionDefragmentationManager {
.futureFor(CheckpointState.FINISHED);
}
+ PageStore oldIdxPageStore =
filePageStoreMgr.getStore(grpId, INDEX_PARTITION);
+
idxDfrgFut = idxDfrgFut.chain(fut -> {
+ if (log.isDebugEnabled()) {
+ log.debug(S.toString(
+ "Index partition defragmented",
+ "grpId", grpId, false,
+ "oldPages", oldIdxPageStore.pages(), false,
+ "newPages", idxAllocationTracker.get() + 1,
false,
+ "pageSize", pageSize, false,
+ "partFile",
defragmentedIndexFile(workDir).getName(), false,
+ "workDir", workDir, false
+ ));
+ }
+
oldPageMem.invalidate(grpId,
PageIdAllocator.INDEX_PARTITION);
PageMemoryEx partPageMem =
(PageMemoryEx)partDataRegion.pageMemory();
@@ -476,8 +491,6 @@ public class CachePartitionDefragmentationManager {
return null;
});
- PageStore oldIdxPageStore =
filePageStoreMgr.getStore(grpId, INDEX_PARTITION);
-
status.onIndexDefragmented(
oldGrpCtx,
oldIdxPageStore.size(),
@@ -614,6 +627,9 @@ public class CachePartitionDefragmentationManager {
CacheDataTree tree = partCtx.oldCacheDataStore.tree();
CacheDataTree newTree = partCtx.newCacheDataStore.tree();
+
+ newTree.enableSequentialWriteMode();
+
PendingEntriesTree newPendingTree =
partCtx.newCacheDataStore.pendingTree();
AbstractFreeList<CacheDataRow> freeList =
partCtx.newCacheDataStore.getCacheStoreFreeList();
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
index c50e75d..10e2c66 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -63,7 +63,6 @@ import
org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoRes
import
org.apache.ignite.internal.processors.cache.persistence.tree.reuse.LongListReuseBag;
import
org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
import
org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import
org.apache.ignite.internal.processors.cache.persistence.tree.util.InsertLast;
import
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
import
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandlerWrapper;
import
org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
@@ -154,6 +153,9 @@ public abstract class BPlusTree<L, T extends L> extends
DataStructure implements
/** Failure processor. */
private final FailureProcessor failureProcessor;
+ /** Flag for enabling single-threaded append-only tree creation. */
+ private boolean sequentialWriteOptsEnabled;
+
/** */
private final GridTreePrinter<Long> treePrinter = new
GridTreePrinter<Long>() {
/** */
@@ -884,6 +886,11 @@ public abstract class BPlusTree<L, T extends L> extends
DataStructure implements
return name;
}
+ /** Flag for enabling single-threaded append-only tree creation. */
+ public void enableSequentialWriteMode() {
+ sequentialWriteOptsEnabled = true;
+ }
+
/**
* Initialize new tree.
*
@@ -1398,6 +1405,8 @@ public abstract class BPlusTree<L, T extends L> extends
DataStructure implements
* @throws IgniteCheckedException If failed.
*/
private void doFind(Get g) throws IgniteCheckedException {
+ assert !sequentialWriteOptsEnabled;
+
for (;;) { // Go down with retries.
g.init();
@@ -2054,6 +2063,8 @@ public abstract class BPlusTree<L, T extends L> extends
DataStructure implements
* @throws IgniteCheckedException If failed.
*/
private T doRemove(L row, boolean needOld) throws IgniteCheckedException {
+ assert !sequentialWriteOptsEnabled;
+
checkDestroyed();
Remove r = new Remove(row, needOld);
@@ -2711,7 +2722,8 @@ public abstract class BPlusTree<L, T extends L> extends
DataStructure implements
long pageId, long page, long pageAddr, BPlusIO io, long fwdId, long
fwdBuf, int idx
) throws IgniteCheckedException {
int cnt = io.getCount(pageAddr);
- int mid = cnt >>> 1;
+
+ int mid = sequentialWriteOptsEnabled ? (int)(cnt * 0.85) : cnt >>> 1;
boolean res = false;
@@ -2767,7 +2779,7 @@ public abstract class BPlusTree<L, T extends L> extends
DataStructure implements
* @return Result code.
* @throws IgniteCheckedException If failed.
*/
- private Result putDown(final Put p, final long pageId, final long fwdId,
final int lvl)
+ private Result putDown(final Put p, final long pageId, final long fwdId,
int lvl)
throws IgniteCheckedException {
assert lvl >= 0 : lvl;
@@ -5302,8 +5314,11 @@ public abstract class BPlusTree<L, T extends L> extends
DataStructure implements
throws IgniteCheckedException {
assert row != null;
- if (row instanceof InsertLast)
+ if (sequentialWriteOptsEnabled) {
+ assert io.getForward(buf) == 0L;
+
return -cnt - 1;
+ }
int high = cnt - 1;
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/InsertLast.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/InsertLast.java
deleted file mode 100644
index e7cfd41..0000000
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/InsertLast.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.tree.util;
-
-/**
- * Rows with this marker interface will always be inserted in the very end of
the tree.
- */
-public interface InsertLast {
-}
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDefragmentationTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDefragmentationTest.java
index 4ce9bb4..25037d3 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDefragmentationTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDefragmentationTest.java
@@ -80,7 +80,7 @@ public class IgnitePdsDefragmentationTest extends
GridCommonAbstractTest {
public static final int PARTS = 5;
/** */
- public static final int ADDED_KEYS_COUNT = 150;
+ public static final int ADDED_KEYS_COUNT = 1500;
/** */
protected static final String GRP_NAME = "group";
diff --git
a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/defragmentation/IndexingDefragmentation.java
b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/defragmentation/IndexingDefragmentation.java
index 19d15c4..050d381 100644
---
a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/defragmentation/IndexingDefragmentation.java
+++
b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/defragmentation/IndexingDefragmentation.java
@@ -40,7 +40,6 @@ import
org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeaf
import
org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import
org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver;
-import
org.apache.ignite.internal.processors.cache.persistence.tree.util.InsertLast;
import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccDataRow;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.h2.database.H2Tree;
@@ -158,6 +157,8 @@ public class IndexingDefragmentation {
for (int i = 0; i < segments; i++) {
H2Tree tree = oldH2Idx.treeForRead(i);
+ newIdx.treeForRead(i).enableSequentialWriteMode();
+
treeIterator.iterate(tree, oldCachePageMem, (theTree, io,
pageAddr, idx) -> {
cancellationChecker.run();
@@ -396,7 +397,7 @@ public class IndexingDefragmentation {
/**
* H2CacheRow with stored index values
*/
- private static class H2CacheRowWithIndex extends H2CacheRow implements
InsertLast {
+ private static class H2CacheRowWithIndex extends H2CacheRow {
/** List of index values. */
private final List<Value> values;
@@ -406,6 +407,7 @@ public class IndexingDefragmentation {
this.values = values;
}
+ /** */
public static H2CacheRowWithIndex create(
GridH2RowDescriptor desc,
long newLink,
