ignite-5918 Adding and searching objects in index tree produces a lot of garbage
(cherry picked from commit 0e0c2c7) (cherry picked from commit 20739d8) Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/71cd1e91 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/71cd1e91 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/71cd1e91 Branch: refs/heads/ignite-gg-12822 Commit: 71cd1e9188f59293c505dedd37c325687f3159f0 Parents: 212603e Author: Igor Seliverstov <[email protected]> Authored: Wed Aug 9 18:46:53 2017 +0300 Committer: Dmitriy Govorukhin <[email protected]> Committed: Fri Sep 22 15:28:26 2017 +0300 ---------------------------------------------------------------------- .../java/org/apache/ignite/IgniteCache.java | 1 - .../query/h2/database/H2TreeIndex.java | 6 +- .../query/h2/database/InlineIndexHelper.java | 414 ++++++++++++++++++- .../h2/database/InlineIndexHelperTest.java | 248 +++++++++-- .../IgniteCacheWithIndexingTestSuite.java | 3 + 5 files changed, 642 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/71cd1e91/modules/core/src/main/java/org/apache/ignite/IgniteCache.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java index 0cf2a82..973eeab 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java @@ -1261,7 +1261,6 @@ public interface IgniteCache<K, V> extends javax.cache.Cache<K, V>, IgniteAsyncS public <T> IgniteFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Set<? extends K> keys, EntryProcessor<K, V, T> entryProcessor, Object... args) throws TransactionException; - /** * Invokes an {@link CacheEntryProcessor} against the set of {@link javax.cache.Cache.Entry}s * specified by the set of keys. http://git-wip-us.apache.org/repos/asf/ignite/blob/71cd1e91/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java index 35bfdc1..de5dc75 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java @@ -145,7 +145,11 @@ public class H2TreeIndex extends GridH2IndexBase { if (!InlineIndexHelper.AVAILABLE_TYPES.contains(col.column.getType())) break; - InlineIndexHelper idx = new InlineIndexHelper(col.column.getType(), col.column.getColumnId(), col.sortType); + InlineIndexHelper idx = new InlineIndexHelper( + col.column.getType(), + col.column.getColumnId(), + col.sortType, + table.getCompareMode()); res.add(idx); } http://git-wip-us.apache.org/repos/asf/ignite/blob/71cd1e91/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java index 19cf857..1789ac8 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java @@ -23,8 +23,10 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; import org.apache.ignite.internal.pagemem.PageUtils; +import org.apache.ignite.internal.util.GridUnsafe; import org.h2.result.SortOrder; import org.h2.table.IndexColumn; +import org.h2.value.CompareMode; import org.h2.value.Value; import org.h2.value.ValueBoolean; import org.h2.value.ValueByte; @@ -84,16 +86,28 @@ public class InlineIndexHelper { /** */ private final short size; + /** */ + private final boolean compareBinaryUnsigned; + + /** */ + private final boolean compareStringsOptimized; + /** * @param type Index type (see {@link Value}). * @param colIdx Index column index. * @param sortType Column sort type (see {@link IndexColumn#sortType}). */ - public InlineIndexHelper(int type, int colIdx, int sortType) { + public InlineIndexHelper(int type, int colIdx, int sortType, CompareMode compareMode) { this.type = type; this.colIdx = colIdx; this.sortType = sortType; + this.compareBinaryUnsigned = compareMode.isBinaryUnsigned(); + + // Optimized strings comparison can be used only if there are no custom collators. + // H2 internal comparison will be used otherwise (may be slower). + this.compareStringsOptimized = CompareMode.OFF.equals(compareMode.getName()); + switch (type) { case Value.BOOLEAN: case Value.BYTE: @@ -330,13 +344,17 @@ public class InlineIndexHelper { * @return Compare result (-2 means we can't compare). */ public int compare(long pageAddr, int off, int maxSize, Value v, Comparator<Value> comp) { + int c = tryCompareOptimized(pageAddr, off, maxSize, v); + + if (c != Integer.MIN_VALUE) + return c; + Value v1 = get(pageAddr, off, maxSize); if (v1 == null) return -2; - int c = comp.compare(v1, v); - c = c != 0 ? c > 0 ? 1 : -1 : 0; + c = Integer.signum(comp.compare(v1, v)); if (size > 0) return fixSort(c, sortType()); @@ -350,6 +368,396 @@ public class InlineIndexHelper { /** * @param pageAddr Page address. * @param off Offset. + * @param maxSize Maximum size to read. + * @param v Value to compare. + * @return Compare result ({@code Integer.MIN_VALUE} means unsupported operation; {@code -2} - can't compare). + */ + private int tryCompareOptimized(long pageAddr, int off, int maxSize, Value v) { + int type; + + if ((size > 0 && size + 1 > maxSize) + || maxSize < 1 + || (type = PageUtils.getByte(pageAddr, off)) == Value.UNKNOWN) + return -2; + + if (type == Value.NULL) + return Integer.MIN_VALUE; + + if (this.type != type) + throw new UnsupportedOperationException("Invalid fast index type: " + type); + + type = Value.getHigherOrder(type, v.getType()); + + switch (type) { + case Value.BOOLEAN: + case Value.BYTE: + case Value.SHORT: + case Value.INT: + case Value.LONG: + case Value.FLOAT: + case Value.DOUBLE: + return compareAsPrimitive(pageAddr, off, v, type); + + case Value.TIME: + case Value.DATE: + case Value.TIMESTAMP: + return compareAsDateTime(pageAddr, off, v, type); + + case Value.STRING: + case Value.STRING_FIXED: + case Value.STRING_IGNORECASE: + if (compareStringsOptimized) + return compareAsString(pageAddr, off, v, type == Value.STRING_IGNORECASE); + + break; + + case Value.BYTES: + return compareAsBytes(pageAddr, off, v); + } + + return Integer.MIN_VALUE; + } + + /** + * @param pageAddr Page address. + * @param off Offset. + * @param v Value to compare. + * @param type Highest value type. + * @return Compare result ({@code -2} means we can't compare). + */ + private int compareAsDateTime(long pageAddr, int off, Value v, int type) { + // only compatible types are supported now. + if(PageUtils.getByte(pageAddr, off) == type) { + switch (type) { + case Value.TIME: + long nanos1 = PageUtils.getLong(pageAddr, off + 1); + long nanos2 = ((ValueTime)v.convertTo(type)).getNanos(); + + return fixSort(Long.signum(nanos1 - nanos2), sortType()); + + case Value.DATE: + long date1 = PageUtils.getLong(pageAddr, off + 1); + long date2 = ((ValueDate)v.convertTo(type)).getDateValue(); + + return fixSort(Long.signum(date1 - date2), sortType()); + + case Value.TIMESTAMP: + ValueTimestamp v0 = (ValueTimestamp) v.convertTo(type); + + date1 = PageUtils.getLong(pageAddr, off + 1); + date2 = v0.getDateValue(); + + int c = Long.signum(date1 - date2); + + if (c == 0) { + nanos1 = PageUtils.getLong(pageAddr, off + 9); + nanos2 = v0.getTimeNanos(); + + c = Long.signum(nanos1 - nanos2); + } + + return fixSort(c, sortType()); + } + } + + return Integer.MIN_VALUE; + } + + /** + * @param pageAddr Page address. + * @param off Offset. + * @param v Value to compare. + * @param type Highest value type. + * @return Compare result ({@code -2} means we can't compare). + */ + private int compareAsPrimitive(long pageAddr, int off, Value v, int type) { + // only compatible types are supported now. + if(PageUtils.getByte(pageAddr, off) == type) { + switch (type) { + case Value.BOOLEAN: + boolean bool1 = PageUtils.getByte(pageAddr, off + 1) != 0; + boolean bool2 = v.getBoolean(); + + return fixSort(Boolean.compare(bool1, bool2), sortType()); + + case Value.BYTE: + byte byte1 = PageUtils.getByte(pageAddr, off + 1); + byte byte2 = v.getByte(); + + return fixSort(Integer.signum(byte1 - byte2), sortType()); + + case Value.SHORT: + short short1 = PageUtils.getShort(pageAddr, off + 1); + short short2 = v.getShort(); + + return fixSort(Integer.signum(short1 - short2), sortType()); + + case Value.INT: + int int1 = PageUtils.getInt(pageAddr, off + 1); + int int2 = v.getInt(); + + return fixSort(Integer.compare(int1, int2), sortType()); + + case Value.LONG: + long long1 = PageUtils.getLong(pageAddr, off + 1); + long long2 = v.getLong(); + + return fixSort(Long.compare(long1, long2), sortType()); + + case Value.FLOAT: + float float1 = Float.intBitsToFloat(PageUtils.getInt(pageAddr, off + 1)); + float float2 = v.getFloat(); + + return fixSort(Float.compare(float1, float2), sortType()); + + case Value.DOUBLE: + double double1 = Double.longBitsToDouble(PageUtils.getLong(pageAddr, off + 1)); + double double2 = v.getDouble(); + + return fixSort(Double.compare(double1, double2), sortType()); + } + } + + return Integer.MIN_VALUE; + } + + /** + * @param pageAddr Page address. + * @param off Offset. + * @param v Value to compare. + * @return Compare result ({@code -2} means we can't compare). + */ + private int compareAsBytes(long pageAddr, int off, Value v) { + byte[] bytes = v.getBytesNoCopy(); + + int len1; + + long addr = pageAddr + off + 1; // Skip type. + + if(size > 0) + // Fixed size value. + len1 = size; + else { + len1 = PageUtils.getShort(pageAddr, off + 1) & 0x7FFF; + + addr += 2; // Skip size. + } + + int len2 = bytes.length; + + int len = Math.min(len1, len2); + + if (compareBinaryUnsigned) { + for (int i = 0; i < len; i++) { + int b1 = GridUnsafe.getByte(addr + i) & 0xff; + int b2 = bytes[i] & 0xff; + + if (b1 != b2) + return fixSort(Integer.signum(b1 - b2), sortType()); + } + } + else { + for (int i = 0; i < len; i++) { + byte b1 = GridUnsafe.getByte(addr + i); + byte b2 = bytes[i]; + + if (b1 != b2) + return fixSort(Integer.signum(b1 - b2), sortType()); + } + } + + int res = Integer.signum(len1 - len2); + + if(isValueFull(pageAddr, off)) + return fixSort(res, sortType()); + + if (res >= 0) + // There are two cases: + // a) The values are equal but the stored value is truncated, so that it's bigger. + // b) Even truncated current value is longer, so that it's bigger. + return fixSort(1, sortType()); + + return -2; + } + + /** + * @param pageAddr Page address. + * @param off Offset. + * @param v Value to compare. + * @param ignoreCase {@code True} if a case-insensitive comparison should be used. + * @return Compare result ({@code -2} means we can't compare). + */ + private int compareAsString(long pageAddr, int off, Value v, boolean ignoreCase) { + String s = v.getString(); + + int len1 = PageUtils.getShort(pageAddr, off + 1) & 0x7FFF; + int len2 = s.length(); + + int c, c2, c3, c4, cntr1 = 0, cntr2 = 0; + char v1, v2; + + long addr = pageAddr + off + 3; // Skip length and type byte. + + // Try reading ASCII. + while (cntr1 < len1 && cntr2 < len2) { + c = (int) GridUnsafe.getByte(addr) & 0xFF; + + if (c > 127) + break; + + cntr1++; addr++; + + v1 = (char)c; + v2 = s.charAt(cntr2++); + + if (ignoreCase) { + v1 = Character.toUpperCase(v1); + v2 = Character.toUpperCase(v2); + } + + if (v1 != v2) + return fixSort(Integer.signum(v1 - v2), sortType()); + } + + // read other + while (cntr1 < len1 && cntr2 < len2) { + c = (int) GridUnsafe.getByte(addr++) & 0xFF; + + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx*/ + cntr1++; + + v1 = (char)c; + + break; + + case 12: + case 13: + /* 110x xxxx 10xx xxxx*/ + cntr1 += 2; + + if (cntr1 > len1) + throw new IllegalStateException("Malformed input (partial character at the end)."); + + c2 = (int) GridUnsafe.getByte(addr++) & 0xFF; + + if ((c2 & 0xC0) != 0x80) + throw new IllegalStateException("Malformed input around byte: " + (cntr1 - 2)); + + c = c & 0x1F; + c = (c << 6) | (c2 & 0x3F); + + v1 = (char)c; + + break; + + case 14: + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + cntr1 += 3; + + if (cntr1 > len1) + throw new IllegalStateException("Malformed input (partial character at the end)."); + + c2 = (int) GridUnsafe.getByte(addr++) & 0xFF; + + c3 = (int) GridUnsafe.getByte(addr++) & 0xFF; + + if (((c2 & 0xC0) != 0x80) || ((c3 & 0xC0) != 0x80)) + throw new IllegalStateException("Malformed input around byte: " + (cntr1 - 3)); + + c = c & 0x0F; + c = (c << 6) | (c2 & 0x3F); + c = (c << 6) | (c3 & 0x3F); + + v1 = (char)c; + + break; + + case 15: + /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ + cntr1 += 4; + + if (cntr1 > len1) + throw new IllegalStateException("Malformed input (partial character at the end)."); + + c2 = (int) GridUnsafe.getByte(addr++) & 0xFF; + + c3 = (int) GridUnsafe.getByte(addr++) & 0xFF; + + c4 = (int) GridUnsafe.getByte(addr++) & 0xFF; + + if (((c & 0xF8) != 0xf0) || ((c2 & 0xC0) != 0x80) || ((c3 & 0xC0) != 0x80) || ((c4 & 0xC0) != 0x80)) + throw new IllegalStateException("Malformed input around byte: " + (cntr1 - 4)); + + c = c & 0x07; + c = (c << 6) | (c2 & 0x3F); + c = (c << 6) | (c3 & 0x3F); + c = (c << 6) | (c4 & 0x3F); + + c = c - 0x010000; // Subtract 0x010000, c is now 0..fffff (20 bits) + + // height surrogate + v1 = (char)(0xD800 + ((c >> 10) & 0x7FF)); + v2 = s.charAt(cntr2++); + + if (v1 != v2) + return fixSort(Integer.signum(v1 - v2), sortType()); + + if (cntr2 == len2) + // The string is malformed (partial partial character at the end). + // Finish comparison here. + return fixSort(1, sortType()); + + // Low surrogate. + v1 = (char)(0xDC00 + (c & 0x3FF)); + v2 = s.charAt(cntr2++); + + if (v1 != v2) + return fixSort(Integer.signum(v1 - v2), sortType()); + + continue; + + default: + /* 10xx xxxx */ + throw new IllegalStateException("Malformed input around byte: " + cntr1); + } + + v2 = s.charAt(cntr2++); + + if (ignoreCase) { + v1 = Character.toUpperCase(v1); + v2 = Character.toUpperCase(v2); + } + + if (v1 != v2) + return fixSort(Integer.signum(v1 - v2), sortType()); + } + + int res = cntr1 == len1 && cntr2 == len2 ? 0 : cntr1 == len1 ? -1 : 1; + + if (isValueFull(pageAddr, off)) + return fixSort(res, sortType()); + + if (res >= 0) + // There are two cases: + // a) The values are equal but the stored value is truncated, so that it's bigger. + // b) Even truncated current value is longer, so that it's bigger. + return fixSort(1, sortType()); + + return -2; + } + + /** + * @param pageAddr Page address. + * @param off Offset. * @param val Value. * @return NUmber of bytes saved. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/71cd1e91/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java index a2a3a72..fc06502 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java @@ -21,8 +21,9 @@ import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; import java.util.Arrays; +import java.util.Comparator; import java.util.UUID; -import junit.framework.TestCase; +import java.util.concurrent.ThreadLocalRandom; import org.apache.commons.io.Charsets; import org.apache.ignite.configuration.MemoryPolicyConfiguration; import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider; @@ -30,7 +31,7 @@ import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl; import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl; -import org.apache.ignite.logger.java.JavaLogger; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.h2.result.SortOrder; import org.h2.value.CompareMode; import org.h2.value.Value; @@ -52,7 +53,7 @@ import org.h2.value.ValueUuid; /** * Simple tests for {@link InlineIndexHelper}. */ -public class InlineIndexHelperTest extends TestCase { +public class InlineIndexHelperTest extends GridCommonAbstractTest { /** */ private static final int CACHE_ID = 42; @@ -79,17 +80,163 @@ public class InlineIndexHelperTest extends TestCase { assertEquals(4, bytes.length); } + /** */ + public void testCompare1bytes() throws Exception { + int maxSize = 3 + 2; // 2 ascii chars + 3 bytes header. + + assertEquals(0, putAndCompare("aa", "aa", maxSize)); + assertEquals(-1, putAndCompare("aa", "bb", maxSize)); + assertEquals(-1, putAndCompare("aaa", "bbb", maxSize)); + assertEquals(1, putAndCompare("bbb", "aaa", maxSize)); + assertEquals(1, putAndCompare("aaa", "aa", maxSize)); + assertEquals(1, putAndCompare("aaa", "a", maxSize)); + assertEquals(-2, putAndCompare("aaa", "aaa", maxSize)); + assertEquals(-2, putAndCompare("aaa", "aab", maxSize)); + assertEquals(-2, putAndCompare("aab", "aaa", maxSize)); + } + + /** */ + public void testCompare2bytes() throws Exception { + int maxSize = 3 + 4; // 2 2-bytes chars + 3 bytes header. + + assertEquals(0, putAndCompare("¡¡", "¡¡", maxSize)); + assertEquals(-1, putAndCompare("¡¡", "¢¢", maxSize)); + assertEquals(-1, putAndCompare("¡¡¡", "¢¢¢", maxSize)); + assertEquals(1, putAndCompare("¢¢¢", "¡¡¡", maxSize)); + assertEquals(1, putAndCompare("¡¡¡", "¡¡", maxSize)); + assertEquals(1, putAndCompare("¡¡¡", "¡", maxSize)); + assertEquals(-2, putAndCompare("¡¡¡", "¡¡¡", maxSize)); + assertEquals(-2, putAndCompare("¡¡¡", "¡¡¢", maxSize)); + assertEquals(-2, putAndCompare("¡¡¢", "¡¡¡", maxSize)); + } + + /** */ + public void testCompare3bytes() throws Exception { + int maxSize = 3 + 6; // 2 3-bytes chars + 3 bytes header. + + assertEquals(0, putAndCompare("à¤à¤", "à¤à¤", maxSize)); + assertEquals(-1, putAndCompare("à¤à¤", "ठठ", maxSize)); + assertEquals(-1, putAndCompare("à¤à¤à¤", "ठठठ", maxSize)); + assertEquals(1, putAndCompare("ठठठ", "à¤à¤à¤", maxSize)); + assertEquals(1, putAndCompare("à¤à¤à¤", "à¤à¤", maxSize)); + assertEquals(1, putAndCompare("à¤à¤à¤", "à¤", maxSize)); + assertEquals(-2, putAndCompare("à¤à¤à¤", "à¤à¤à¤", maxSize)); + assertEquals(-2, putAndCompare("à¤à¤à¤", "à¤à¤à¤ ", maxSize)); + assertEquals(-2, putAndCompare("à¤à¤à¤ ", "à¤à¤à¤", maxSize)); + } + + /** */ + public void testCompare4bytes() throws Exception { + int maxSize = 3 + 8; // 2 4-bytes chars + 3 bytes header. + + assertEquals(0, putAndCompare("\ud802\udd20\ud802\udd20", "\ud802\udd20\ud802\udd20", maxSize)); + assertEquals(-1, putAndCompare("\ud802\udd20\ud802\udd20", "\ud802\udd21\ud802\udd21", maxSize)); + assertEquals(-1, putAndCompare("\ud802\udd20\ud802\udd20\ud802\udd20", "\ud802\udd21\ud802\udd21\ud802\udd21", maxSize)); + assertEquals(1, putAndCompare("\ud802\udd21\ud802\udd21\ud802\udd21", "\ud802\udd20\ud802\udd20\ud802\udd20", maxSize)); + assertEquals(1, putAndCompare("\ud802\udd20\ud802\udd20\ud802\udd20", "\ud802\udd20\ud802\udd20", maxSize)); + assertEquals(1, putAndCompare("\ud802\udd20\ud802\udd20\ud802\udd20", "\ud802\udd20", maxSize)); + assertEquals(-2, putAndCompare("\ud802\udd20\ud802\udd20\ud802\udd20", "\ud802\udd20\ud802\udd20\ud802\udd20", maxSize)); + assertEquals(-2, putAndCompare("\ud802\udd20\ud802\udd20\ud802\udd20", "\ud802\udd20\ud802\udd20\ud802\udd21", maxSize)); + assertEquals(-2, putAndCompare("\ud802\udd20\ud802\udd20\ud802\udd21", "\ud802\udd20\ud802\udd20\ud802\udd20", maxSize)); + } + + /** */ + public void testCompareMixed() throws Exception { + int maxSize = 3 + 8; // 2 up to 4-bytes chars + 3 bytes header. + + assertEquals(0, putAndCompare("\ud802\udd20\u0904", "\ud802\udd20\u0904", maxSize)); + assertEquals(-1, putAndCompare("\ud802\udd20\u0904", "\ud802\udd20\u0905", maxSize)); + assertEquals(1, putAndCompare("\u0905\ud802\udd20", "\u0904\ud802\udd20", maxSize)); + assertEquals(-2, putAndCompare("\ud802\udd20\ud802\udd20\u0905", "\ud802\udd20\ud802\udd20\u0904", maxSize)); + } + + /** */ + public void testCompareMixed2() throws Exception { + int strCnt = 1000; + int symbCnt = 20; + int inlineSize = symbCnt * 4 + 3; + + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + String[] strings = new String[strCnt]; + + for (int i = 0; i < strings.length; i++) + strings[i] = randomString(symbCnt); + + Arrays.sort(strings); + + for (int i = 0; i < 100; i++) { + int i1 = rnd.nextInt(strings.length); + int i2 = rnd.nextInt(strings.length); + + assertEquals(Integer.compare(i1,i2), putAndCompare(strings[i1], strings[i2], inlineSize)); + } + } + + /** + * @param v1 Value 1. + * @param v2 Value 2. + * @param maxSize Max inline size. + * @return Compare result. + * @throws Exception If failed. + */ + private int putAndCompare(String v1, String v2, int maxSize) throws Exception { + MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration().setInitialSize(1024 * MB) + .setMaxSize(1024 * MB); + + PageMemory pageMem = new PageMemoryNoStoreImpl(log, + new UnsafeMemoryProvider(log), + null, + PAGE_SIZE, + plcCfg, + new MemoryMetricsImpl(plcCfg), + false); + + pageMem.start(); + + long pageId = 0L; + long page = 0L; + + try { + pageId = pageMem.allocatePage(CACHE_ID, 1, PageIdAllocator.FLAG_DATA); + page = pageMem.acquirePage(CACHE_ID, pageId); + long pageAddr = pageMem.readLock(CACHE_ID, pageId, page); + + int off = 0; + + InlineIndexHelper ih = new InlineIndexHelper(Value.STRING, 1, 0, + CompareMode.getInstance(null, 0)); + + ih.put(pageAddr, off, ValueString.get(v1), maxSize); + + Comparator<Value> comp = new Comparator<Value>() { + @Override public int compare(Value o1, Value o2) { + throw new AssertionError("Optimized algorithm should be used."); + } + }; + + return ih.compare(pageAddr, off, maxSize, ValueString.get(v2), comp); + } + finally { + if (page != 0L) + pageMem.releasePage(CACHE_ID, pageId, page); + + pageMem.stop(); + } + } + /** Limit is too small to cut */ public void testStringCut() { // 6 bytes total: 3b, 3b. - byte[] bytes = InlineIndexHelper.trimUTF8("\u20ac\u20ac".getBytes(Charsets.UTF_8), 2); + assertNull(bytes); } /** Test on String values compare */ public void testRelyOnCompare() { - InlineIndexHelper ha = new InlineIndexHelper(Value.STRING, 0, SortOrder.ASCENDING); + InlineIndexHelper ha = new InlineIndexHelper(Value.STRING, 0, SortOrder.ASCENDING, + CompareMode.getInstance(null, 0)); // same size assertFalse(getRes(ha, "aabb", "aabb")); @@ -110,7 +257,8 @@ public class InlineIndexHelperTest extends TestCase { /** Test on Bytes values compare */ public void testRelyOnCompareBytes() { - InlineIndexHelper ha = new InlineIndexHelper(Value.BYTES, 0, SortOrder.ASCENDING); + InlineIndexHelper ha = new InlineIndexHelper(Value.BYTES, 0, SortOrder.ASCENDING, + CompareMode.getInstance(null, 0)); // same size assertFalse(getResBytes(ha, new byte[] {1, 2, 3, 4}, new byte[] {1, 2, 3, 4})); @@ -131,12 +279,11 @@ public class InlineIndexHelperTest extends TestCase { /** */ public void testStringTruncate() throws Exception { - MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration().setMaxSize(1024 * MB); - - JavaLogger log = new JavaLogger(); + MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration().setInitialSize(1024 * MB) + .setMaxSize(1024 * MB); - PageMemory pageMem = new PageMemoryNoStoreImpl(log, - new UnsafeMemoryProvider(log), + PageMemory pageMem = new PageMemoryNoStoreImpl(log(), + new UnsafeMemoryProvider(log()), null, PAGE_SIZE, plcCfg, @@ -155,7 +302,9 @@ public class InlineIndexHelperTest extends TestCase { int off = 0; - InlineIndexHelper ih = new InlineIndexHelper(Value.STRING, 1, 0); + InlineIndexHelper ih = new InlineIndexHelper(Value.STRING, 1, 0, + CompareMode.getInstance(null, 0)); + ih.put(pageAddr, off, ValueString.get("aaaaaaa"), 3 + 5); assertFalse(ih.isValueFull(pageAddr, off)); @@ -181,12 +330,11 @@ public class InlineIndexHelperTest extends TestCase { /** */ public void testBytes() throws Exception { - MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration().setMaxSize(1024 * MB); + MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration().setInitialSize(1024 * MB) + .setMaxSize(1024 * MB); - JavaLogger log = new JavaLogger(); - - PageMemory pageMem = new PageMemoryNoStoreImpl(log, - new UnsafeMemoryProvider(log), + PageMemory pageMem = new PageMemoryNoStoreImpl(log(), + new UnsafeMemoryProvider(log()), null, PAGE_SIZE, plcCfg, @@ -205,7 +353,8 @@ public class InlineIndexHelperTest extends TestCase { int off = 0; - InlineIndexHelper ih = new InlineIndexHelper(Value.BYTES, 1, 0); + InlineIndexHelper ih = new InlineIndexHelper(Value.BYTES, 1, 0, + CompareMode.getInstance(null, 0)); ih.put(pageAddr, off, ValueBytes.get(new byte[] {1, 2, 3, 4, 5}), 3 + 3); @@ -296,12 +445,11 @@ public class InlineIndexHelperTest extends TestCase { /** */ private void testPutGet(Value v1, Value v2, Value v3) throws Exception { - MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration().setMaxSize(1024 * MB); + MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration().setInitialSize(1024 * MB) + .setMaxSize(1024 * MB); - JavaLogger log = new JavaLogger(); - - PageMemory pageMem = new PageMemoryNoStoreImpl(log, - new UnsafeMemoryProvider(log), + PageMemory pageMem = new PageMemoryNoStoreImpl(log(), + new UnsafeMemoryProvider(log()), null, PAGE_SIZE, plcCfg, @@ -321,7 +469,8 @@ public class InlineIndexHelperTest extends TestCase { int off = 0; int max = 255; - InlineIndexHelper ih = new InlineIndexHelper(v1.getType(), 1, 0); + InlineIndexHelper ih = new InlineIndexHelper(v1.getType(), 1, 0, + CompareMode.getInstance(null, 0)); off += ih.put(pageAddr, off, v1, max - off); off += ih.put(pageAddr, off, v2, max - off); @@ -336,6 +485,7 @@ public class InlineIndexHelperTest extends TestCase { finally { if (page != 0L) pageMem.releasePage(CACHE_ID, pageId, page); + pageMem.stop(); } } @@ -346,6 +496,7 @@ public class InlineIndexHelperTest extends TestCase { Value v2 = s2 == null ? ValueNull.INSTANCE : ValueString.get(s2); int c = v1.compareTypeSafe(v2, CompareMode.getInstance(CompareMode.DEFAULT, 0)); + return ha.canRelyOnCompare(c, v1, v2); } @@ -355,7 +506,54 @@ public class InlineIndexHelperTest extends TestCase { Value v2 = b2 == null ? ValueNull.INSTANCE : ValueBytes.get(b2); int c = v1.compareTypeSafe(v2, CompareMode.getInstance(CompareMode.DEFAULT, 0)); + return ha.canRelyOnCompare(c, v1, v2); } -} \ No newline at end of file + /** + * @param cnt String length. + * @return Random string. + */ + private String randomString(int cnt) { + final char[] buffer = new char[cnt]; + + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + while (cnt-- != 0) { + char ch; + + if (rnd.nextInt(100) > 3) + ch = (char) (rnd.nextInt(95) + 32); // regular symbols + else + ch = (char) (rnd.nextInt(65407) + 127); // others symbols + + if(ch >= 56320 && ch <= 57343) { + if(cnt == 0) + cnt++; + else { + // low surrogate, insert high surrogate after putting it in + buffer[cnt] = ch; + cnt--; + buffer[cnt] = (char) (55296 + rnd.nextInt(128)); + } + } + else if(ch >= 55296 && ch <= 56191) { + if(cnt == 0) + cnt++; + else { + // high surrogate, insert low surrogate before putting it in + buffer[cnt] = (char) (56320 + rnd.nextInt(128)); + cnt--; + buffer[cnt] = ch; + } + } + else if(ch >= 56192 && ch <= 56319) + // private high surrogate, no effing clue, so skip it + cnt++; + else + buffer[cnt] = ch; + } + + return new String(buffer); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/71cd1e91/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java index 794ec4d..4047700 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java @@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicLocalSelfTe import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicPartitionedSelfTest; import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalLocalSelfTest; import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalPartitionedSelfTest; +import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelperTest; /** * Cache tests using indexing. @@ -49,6 +50,8 @@ public class IgniteCacheWithIndexingTestSuite extends TestSuite { public static TestSuite suite() throws Exception { TestSuite suite = new TestSuite("Ignite Cache With Indexing Test Suite"); + suite.addTestSuite(InlineIndexHelperTest.class); + suite.addTestSuite(GridIndexingWithNoopSwapSelfTest.class); suite.addTestSuite(GridCacheOffHeapSelfTest.class);
