Repository: kylin Updated Branches: refs/heads/master 2309618b9 -> 7e213b013
KYLIN-2956 building trie dictionary blocked on value of length over 4095 Signed-off-by: lidongsjtu <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7e213b01 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7e213b01 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7e213b01 Branch: refs/heads/master Commit: 7e213b0132e4c53a396755f75750ac7e48e084c4 Parents: 2309618 Author: gwang3 <[email protected]> Authored: Thu Dec 21 14:58:42 2017 +0800 Committer: lidongsjtu <[email protected]> Committed: Thu Dec 21 15:06:32 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/BytesUtil.java | 2 +- .../apache/kylin/dict/TrieDictionaryTest.java | 30 +++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7e213b01/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java index d41c6b4..6b64bbe 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java @@ -178,7 +178,7 @@ public class BytesUtil { } public static boolean isPositiveShort(int i) { - return (i & 0xFFFF7000) == 0; + return (i & 0xFFFF8000) == 0; } // from WritableUtils http://git-wip-us.apache.org/repos/asf/kylin/blob/7e213b01/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java index 13c83ac..600b072 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java @@ -39,6 +39,7 @@ import java.util.NoSuchElementException; import java.util.Random; import java.util.TreeSet; +import org.junit.Assert; import org.junit.Test; public class TrieDictionaryTest { @@ -408,13 +409,34 @@ public class TrieDictionaryTest { String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"; TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); - String v1 = longPrefix + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; - String v2 = longPrefix + "xyz"; - + String v1 = longPrefix + "xyz"; b.addValue(v1); - b.addValue(v2); + + String strLen200 = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghid"; + b.addValue(strLen200); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 25; i++) { + sb.append(strLen200); + } + String strLen5000 = sb.toString(); + b.addValue(strLen5000); TrieDictionary<String> dict = b.build(0); dict.dump(System.out); + + sb.setLength(0); + for (int j = 0; j < 7; j++) { + sb.append(strLen5000); + } + String strLen35000 = sb.toString(); + b.addValue(strLen35000); + Exception ex = null; + try { + b.build(0); + } catch (Exception e) { + ex = e; + } + Assert.assertNotNull(ex); } @Test
