Ignite-db - Fixes.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e52e37e7 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e52e37e7 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e52e37e7 Branch: refs/heads/ignite-db-x-10884 Commit: e52e37e7dceb10467ee9c398e7d47fe9c32de449 Parents: afc76c8 Author: Alexey Goncharuk <[email protected]> Authored: Sat Apr 30 20:20:39 2016 -0700 Committer: Alexey Goncharuk <[email protected]> Committed: Sat Apr 30 20:20:39 2016 -0700 ---------------------------------------------------------------------- .../cache/database/MetadataStorage.java | 2 +- .../cache/database/tree/io/DataPageIO.java | 59 +++++++++++++++++--- .../query/h2/database/H2RowStore.java | 7 --- .../processors/query/h2/opt/GridH2Table.java | 18 +++--- .../IgniteDbSingleNodePutGetSelfTest.java | 50 ++++++++++++++++- 5 files changed, 109 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/e52e37e7/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java index 047c5f1..e0eccca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java @@ -44,7 +44,7 @@ public class MetadataStorage implements MetaStore { } /** {@inheritDoc} */ - @Override public IgniteBiTuple<FullPageId, Boolean> getOrAllocateForIndex(int cacheId, String idxName) + @Override public synchronized IgniteBiTuple<FullPageId, Boolean> getOrAllocateForIndex(int cacheId, String idxName) throws IgniteCheckedException { byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8); http://git-wip-us.apache.org/repos/asf/ignite/blob/e52e37e7/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java index 54df845..c3c37f2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java @@ -239,6 +239,7 @@ public class DataPageIO extends PageIO { public String printPageLayout(ByteBuffer buf) { int directCnt = getDirectCount(buf); int indirectCnt = getIndirectCount(buf); + int free = getFreeSpace(buf); boolean valid = directCnt >= indirectCnt; @@ -246,6 +247,8 @@ public class DataPageIO extends PageIO { b.appendHex(PageIO.getPageId(buf)).a(" ["); + int entriesSize = 0; + for (int i = 0; i < directCnt; i++) { if (i != 0) b.a(", "); @@ -255,6 +258,8 @@ public class DataPageIO extends PageIO { if (item < ITEMS_OFF || item >= buf.capacity()) valid = false; + entriesSize += getEntrySize(buf, item, false); + b.a(item); } @@ -286,7 +291,17 @@ public class DataPageIO extends PageIO { b.a(itemId).a('^').a(directIdx); } - b.a("]"); + b.a("][free=").a(free); + + int actualFree = buf.capacity() - ITEMS_OFF - (entriesSize + (directCnt + indirectCnt) * ITEM_SIZE); + + if (free != actualFree) { + b.a(", actualFree=").a(actualFree); + + valid = false; + } + else + b.a("]"); assert valid : b.toString(); @@ -457,8 +472,11 @@ public class DataPageIO extends PageIO { public void removeRow(ByteBuffer buf, int itemId) { assert check(itemId) : itemId; - int directCnt = getDirectCount(buf); - int indirectCnt = getIndirectCount(buf); + // Record original counts to calculate delta in free space in the end of remove. + final int directCnt = getDirectCount(buf); + final int indirectCnt = getIndirectCount(buf); + + int curIndirectCnt = indirectCnt; assert directCnt > 0 : directCnt; // Direct count always represents overall number of live items. @@ -496,7 +514,7 @@ public class DataPageIO extends PageIO { if (dropLast) moveItems(buf, directCnt, indirectCnt, -1); else - indirectCnt++; + curIndirectCnt++; } else { if (dropLast) @@ -505,10 +523,10 @@ public class DataPageIO extends PageIO { moveItems(buf, indirectId + 1, directCnt + indirectCnt - indirectId - 1, dropLast ? -2 : -1); if (dropLast) - indirectCnt--; + curIndirectCnt--; } - setIndirectCount(buf, indirectCnt); + setIndirectCount(buf, curIndirectCnt); setDirectCount(buf, directCnt - 1); assert getIndirectCount(buf) <= getDirectCount(buf); @@ -599,8 +617,8 @@ public class DataPageIO extends PageIO { assert check(itemId): itemId; assert getIndirectCount(buf) <= getDirectCount(buf); - // Update free space. If number of direct items did not change, then we were able to reuse item slot. - setFreeSpace(buf, getFreeSpace(buf) - entrySizeWithItem + (getDirectCount(buf) == directCnt ? ITEM_SIZE : 0)); + // Update free space. If number of indirect items changed, then we were able to reuse an item slot. + setFreeSpace(buf, getFreeSpace(buf) - entrySizeWithItem + (getIndirectCount(buf) != indirectCnt ? ITEM_SIZE : 0)); assert getFreeSpace(buf) >= 0; @@ -692,6 +710,28 @@ public class DataPageIO extends PageIO { } /** + * Full-scan free space calculation procedure. + * + * @param buf Buffer to scan. + * @return Actual free space in the buffer. + */ + private int actualFreeSpace(ByteBuffer buf) { + int directCnt = getDirectCount(buf); + + int entriesSize = 0; + + for (int i = 0; i < directCnt; i++) { + int off = toOffset(getItem(buf, i)); + + int entrySize = getEntrySize(buf, off, false); + + entriesSize += entrySize; + } + + return buf.capacity() - ITEMS_OFF - entriesSize - (directCnt + getIndirectCount(buf)) * ITEM_SIZE; + } + + /** * @param buf Buffer. * @param off Offset. * @param cnt Count. @@ -700,7 +740,8 @@ public class DataPageIO extends PageIO { private static void moveBytes(ByteBuffer buf, int off, int cnt, int step) { assert step != 0: step; assert off + step >= 0; - assert off + step + cnt < buf.capacity(); + assert off + step + cnt <= buf.capacity() : "[off=" + off + ", step=" + step + ", cnt=" + cnt + + ", cap=" + buf.capacity() + ']'; PageHandler.copyMemory(buf, buf, off, off + step, cnt); } http://git-wip-us.apache.org/repos/asf/ignite/blob/e52e37e7/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowStore.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowStore.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowStore.java index cd0cc4b..54a6c51 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowStore.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowStore.java @@ -66,11 +66,6 @@ public class H2RowStore extends RowStore<GridH2Row> { ByteBuffer buf = page.getForRead(); try { - GridH2Row existing = rowDesc.cachedRow(link); - - if (existing != null) - return existing; - DataPageIO io = DataPageIO.VERSIONS.forPage(buf); int dataOff = io.getDataOffset(buf, dwordsOffset(link)); @@ -103,8 +98,6 @@ public class H2RowStore extends RowStore<GridH2Row> { assert row.ver != null; - rowDesc.cache(row); - return row; } finally { http://git-wip-us.apache.org/repos/asf/ignite/blob/e52e37e7/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java index e149b9d..ad56b54 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java @@ -446,7 +446,7 @@ public class GridH2Table extends TableBase { if (old2 != null) { // Row was replaced in index. if (!eq(pk, old2, old)) throw new IllegalStateException("Row conflict should never happen, unique indexes are " + - "not supported."); + "not supported [idx=" + idx + ", old=" + old + ", old2=" + old2 + ']'); } else if (old != null) // Row was not replaced, need to remove manually. idx.remove(old); @@ -464,14 +464,6 @@ public class GridH2Table extends TableBase { } if (old != null) { - if (rowStore != null) { - assert old.link != 0; - - rowStore.removeRow(old.link); - } - - size.decrement(); - // Remove row from all indexes. // Start from 2 because 0 - Scan (don't need to update), 1 - PK (already updated). for (int i = 2, len = idxs.size(); i < len; i++) { @@ -479,6 +471,14 @@ public class GridH2Table extends TableBase { assert eq(pk, res, old): "\n" + old + "\n" + res + "\n" + i + " -> " + index(i).getName(); } + + if (rowStore != null) { + assert old.link != 0; + + rowStore.removeRow(old.link); + } + + size.decrement(); } else return false; http://git-wip-us.apache.org/repos/asf/ignite/blob/e52e37e7/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java index 0a6c1c9..7536075 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java @@ -11,9 +11,12 @@ package org.apache.ignite.internal.processors.database; import java.io.Serializable; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; + import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.CacheAtomicityMode; @@ -768,6 +771,51 @@ public class IgniteDbSingleNodePutGetSelfTest extends GridCommonAbstractTest { /** * @throws Exception if failed. */ + public void testIndexOverwrite() throws Exception { + IgniteEx ig = grid(0); + + final IgniteCache<Integer, DbValue> cache = ig.cache(null); + + GridCacheAdapter<Object, Object> internalCache = ig.context().cache().internalCache("non-primitive"); + + X.println("Put start"); + + int cnt = 10_000; + + for (int a = 0; a < cnt; a++) { + DbValue v0 = new DbValue(a, "test-value-" + a, a); + + DbKey k0 = new DbKey(a); + + cache.put(a, v0); + + checkEmpty(internalCache, k0); + } + + info("Update start"); + + for (int k = 0; k < 4000; k++) { + int batchSize = 20; + + LinkedHashMap<Integer, DbValue> batch = new LinkedHashMap<>(); + + for (int i = 0; i < batchSize; i++) { + int a = ThreadLocalRandom.current().nextInt(cnt); + + DbValue v0 = new DbValue(a, "test-value-" + a, a); + + batch.put(a, v0); + } + + cache.putAll(batch); + + cache.remove(ThreadLocalRandom.current().nextInt(cnt)); + } + } + + /** + * @throws Exception if failed. + */ public void testObjectKey() throws Exception { IgniteEx ig = grid(0); @@ -859,7 +907,7 @@ public class IgniteDbSingleNodePutGetSelfTest extends GridCommonAbstractTest { private int iVal; /** */ - @QuerySqlField + @QuerySqlField(index = true) private String sVal; /** */
