http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java ---------------------------------------------------------------------- diff --git a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java deleted file mode 100644 index 0af7dfc..0000000 --- a/dictionary/src/test/java/org/apache/kylin/dict/LookupTableTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict; - -import java.io.File; - -import org.apache.kylin.common.util.Array; -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.dict.lookup.FileTable; -import org.apache.kylin.dict.lookup.LookupBytesTable; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.TableDesc; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * @author yangli9 - */ -public class LookupTableTest extends LocalFileMetadataTestCase { - - @Before - public void setup() throws Exception { - createTestMetadata(); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - @Test - public void testBasic() throws Exception { - TableDesc siteTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_SITES"); - TableDesc categoryTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.test_category_groupings"); - LookupBytesTable lookup; - - System.out.println("============================================================================"); - - File f = new File(LOCALMETA_TEST_DATA + "/data/EDW.TEST_SITES.csv"); - lookup = new LookupBytesTable(siteTable, new String[] { "SITE_ID" }, new FileTable("file://" + f.getAbsolutePath(), 10)); - lookup.dump(); - - System.out.println("============================================================================"); - - f = new File(LOCALMETA_TEST_DATA + "/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv"); - lookup = new LookupBytesTable(categoryTable, new String[] { "leaf_categ_id", "site_id" }, new FileTable("file://" + f.getAbsolutePath(), 36)); - lookup.dump(); - - System.out.println("============================================================================"); - - ByteArray k1 = new ByteArray(Bytes.toBytes("533")); - ByteArray k2 = new ByteArray(Bytes.toBytes("0")); - Array<ByteArray> key = new Array<ByteArray>(new ByteArray[] { k1, k2 }); - System.out.println(lookup.getRow(key)); - } -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java ---------------------------------------------------------------------- diff --git a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java deleted file mode 100644 index 4647ef8..0000000 --- a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import java.math.BigDecimal; -import java.util.Collections; -import java.util.List; -import java.util.Random; -import java.util.Set; - -import org.apache.kylin.common.util.Bytes; -import org.junit.Test; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -/** - * @author yangli9 - * - */ -public class NumberDictionaryTest { - - NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec(); - Random rand = new Random(); - - @Test - public void testNumberEncode() { - checkCodec("12345", "00000000000012345"); - checkCodec("12345.123", "00000000000012345.123"); - checkCodec("-12345", "-9999999999987654;"); - checkCodec("-12345.123", "-9999999999987654.876;"); - checkCodec("0", "00000000000000000"); - checkCodec("0.0", "00000000000000000.0"); - } - - private void checkCodec(String number, String code) { - assertEquals(code, encodeNumber(number)); - assertEquals(number, decodeNumber(code)); - } - - private String decodeNumber(String code) { - byte[] buf = Bytes.toBytes(code); - System.arraycopy(buf, 0, codec.buf, 0, buf.length); - codec.bufOffset = 0; - codec.bufLen = buf.length; - int len = codec.decodeNumber(buf, 0); - return Bytes.toString(buf, 0, len); - } - - private String encodeNumber(String number) { - byte[] num1 = Bytes.toBytes(number); - codec.encodeNumber(num1, 0, num1.length); - return Bytes.toString(codec.buf, codec.bufOffset, codec.bufLen); - } - - @Test - public void testDictionary() { - int n = 100; - - Set<BigDecimal> set = Sets.newHashSet(); - NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new StringBytesConverter()); - for (int i = 0; i < n; i++) { - String num = randNumber(); - if (set.add(new BigDecimal(num))) { - builder.addValue(num); - } - } - - List<BigDecimal> sorted = Lists.newArrayList(); - sorted.addAll(set); - Collections.sort(sorted); - - // test exact match - NumberDictionary<String> dict = builder.build(0); - for (int i = 0; i < sorted.size(); i++) { - String dictNum = dict.getValueFromId(i); - System.out.println(sorted.get(i) + "\t" + dictNum); - } - - for (int i = 0; i < sorted.size(); i++) { - String dictNum = dict.getValueFromId(i); - assertEquals(sorted.get(i), new BigDecimal(dictNum)); - } - - // test rounding - for (int i = 0; i < n; i++) { - String randStr = randNumber(); - BigDecimal rand = new BigDecimal(randStr); - int binarySearch = Collections.binarySearch(sorted, rand); - if (binarySearch >= 0) - continue; - int insertion = -(binarySearch + 1); - int expectedLowerId = insertion - 1; - int expectedHigherId = insertion; - // System.out.println("-- " + randStr + ", " + expectedLowerId + - // ", " + expectedHigherId); - - if (expectedLowerId < 0) { - try { - dict.getIdFromValue(randStr, -1); - fail(); - } catch (IllegalArgumentException ex) { - // expect - } - } else { - assertEquals(expectedLowerId, dict.getIdFromValue(randStr, -1)); - } - - if (expectedHigherId >= sorted.size()) { - try { - dict.getIdFromValue(randStr, 1); - fail(); - } catch (IllegalArgumentException ex) { - // expect - } - } else { - assertEquals(expectedHigherId, dict.getIdFromValue(randStr, 1)); - } - } - } - - private String randNumber() { - int digits1 = rand.nextInt(10); - int digits2 = rand.nextInt(3); - int sign = rand.nextInt(2); - if (digits1 == 0 && digits2 == 0) { - return randNumber(); - } - StringBuilder buf = new StringBuilder(); - if (sign == 1) - buf.append("-"); - for (int i = 0; i < digits1; i++) - buf.append("" + rand.nextInt(10)); - if (digits2 > 0) { - buf.append("."); - for (int i = 0; i < digits2; i++) - buf.append("" + rand.nextInt(9) + 1); // BigDecimal thinks 4.5 - // != 4.50, my god! - } - return buf.toString(); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java ---------------------------------------------------------------------- diff --git a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java deleted file mode 100644 index c8dddde..0000000 --- a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import org.apache.kylin.common.util.HBaseMetadataTestCase; -import org.apache.kylin.dict.lookup.HiveTable; -import org.apache.kylin.dict.lookup.ReadableTable.TableReader; -import org.apache.kylin.dict.lookup.SnapshotManager; -import org.apache.kylin.dict.lookup.SnapshotTable; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.TableDesc; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * @author yangli9 - * - */ -public class SnapshotManagerTest extends HBaseMetadataTestCase { - - SnapshotManager snapshotMgr; - - @Before - public void setup() throws Exception { - createTestMetadata(); - snapshotMgr = SnapshotManager.getInstance(getTestConfig()); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - @Test - public void basicTest() throws Exception { - String tableName = "EDW.TEST_SITES"; - HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName); - TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName); - String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath(); - - snapshotMgr.wipeoutCache(); - - SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath); - - // compare hive & snapshot - TableReader hiveReader = hiveTable.getReader(); - TableReader snapshotReader = snapshot.getReader(); - - while (true) { - boolean hiveNext = hiveReader.next(); - boolean snapshotNext = snapshotReader.next(); - assertEquals(hiveNext, snapshotNext); - - if (hiveNext == false) - break; - - String[] hiveRow = hiveReader.getRow(); - String[] snapshotRow = snapshotReader.getRow(); - assertArrayEquals(hiveRow, snapshotRow); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java ---------------------------------------------------------------------- diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java deleted file mode 100644 index 0d1999a..0000000 --- a/dictionary/src/test/java/org/apache/kylin/dict/TableReaderTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - -import org.apache.kylin.dict.lookup.FileTable; -import org.apache.kylin.dict.lookup.FileTableReader; -import org.junit.Test; - -/** - * @author yangli9 - * - */ -public class TableReaderTest { - - @Test - public void testBasicReader() throws IOException { - File f = new File("src/test/resources/dict/DW_SITES"); - FileTableReader reader = new FileTableReader("file://" + f.getAbsolutePath(), FileTable.DELIM_AUTO, 10); - while (reader.next()) { - assertEquals("[-1, Korea Auction.co.kr, S, 48, 0, 111, 2009-02-11, , DW_OFFPLAT, ]", Arrays.toString(reader.getRow())); - break; - } - reader.close(); - - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java ---------------------------------------------------------------------- diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java deleted file mode 100644 index cb15731..0000000 --- a/dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Random; -import java.util.TreeSet; - -import org.junit.Test; - -public class TrieDictionaryTest { - - public static void main(String[] args) throws Exception { - InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat"); - // InputStream is = - // Util.getPackageResourceAsStream(TrieDictionaryTest.class, - // "eng_com.dic"); - ArrayList<String> str = loadStrings(is); - benchmarkStringDictionary(str); - } - - @Test - public void partOverflowTest() { - ArrayList<String> str = new ArrayList<String>(); - // str.add(""); - str.add("part"); - str.add("par"); - str.add("partition"); - str.add("party"); - str.add("parties"); - str.add("paint"); - String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"; - System.out.println("The length of the long string is " + longStr.length()); - str.add(longStr); - - str.add("zzzzzz" + longStr);// another long string - - TrieDictionaryBuilder<String> b = newDictBuilder(str); - TrieDictionary<String> dict = b.build(0); - - TreeSet<String> set = new TreeSet<String>(); - for (String s : str) { - set.add(s); - } - - // test serialize - dict = testSerialize(dict); - - // test basic id<==>value - Iterator<String> it = set.iterator(); - int id = 0; - int previousId = -1; - for (; it.hasNext(); id++) { - String value = it.next(); - - // in case of overflow parts, there exist interpolation nodes - // they exist to make sure that any node's part is shorter than 255 - int actualId = dict.getIdFromValue(value); - assertTrue(actualId >= id); - assertTrue(actualId > previousId); - previousId = actualId; - - assertEquals(value, dict.getValueFromId(actualId)); - } - } - - @Test - public void emptyValueTest() { - ArrayList<String> str = new ArrayList<String>(); - str.add(""); - str.add("part"); - str.add("par"); - str.add("partition"); - str.add("party"); - str.add("parties"); - str.add("paint"); - testStringDictionary(str, null); - } - - @Test - public void simpleTrieTest() { - ArrayList<String> str = new ArrayList<String>(); - str.add("part"); - str.add("part"); // meant to be dup - str.add("par"); - str.add("partition"); - str.add("party"); - str.add("parties"); - str.add("paint"); - - ArrayList<String> notFound = new ArrayList<String>(); - notFound.add(""); - notFound.add("p"); - notFound.add("pa"); - notFound.add("pb"); - notFound.add("parti"); - notFound.add("partz"); - notFound.add("partyz"); - - testStringDictionary(str, notFound); - } - - @Test - public void englishWordsTest() throws Exception { - InputStream is = new FileInputStream("src/test/resources/dict/english-words.80 (scowl-2015.05.18).dic"); - ArrayList<String> str = loadStrings(is); - testStringDictionary(str, null); - } - - @Test - public void categoryNamesTest() throws Exception { - InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat"); - ArrayList<String> str = loadStrings(is); - testStringDictionary(str, null); - } - - private static void benchmarkStringDictionary(ArrayList<String> str) throws UnsupportedEncodingException { - TrieDictionaryBuilder<String> b = newDictBuilder(str); - b.stats().print(); - TrieDictionary<String> dict = b.build(0); - - TreeSet<String> set = new TreeSet<String>(); - for (String s : str) { - set.add(s); - } - - // prepare id==>value array and value==>id map - HashMap<String, Integer> map = new HashMap<String, Integer>(); - String[] strArray = new String[set.size()]; - byte[][] array = new byte[set.size()][]; - Iterator<String> it = set.iterator(); - for (int id = 0; it.hasNext(); id++) { - String value = it.next(); - map.put(value, id); - strArray[id] = value; - array[id] = value.getBytes("UTF-8"); - } - - // System.out.println("Dict size in bytes: " + - // MemoryUtil.deepMemoryUsageOf(dict)); - // System.out.println("Map size in bytes: " + - // MemoryUtil.deepMemoryUsageOf(map)); - // System.out.println("Array size in bytes: " + - // MemoryUtil.deepMemoryUsageOf(strArray)); - - // warm-up, said that code only got JIT after run 1k-10k times, - // following jvm options may help - // -XX:CompileThreshold=1500 - // -XX:+PrintCompilation - benchmark("Warm up", dict, set, map, strArray, array); - benchmark("Benchmark", dict, set, map, strArray, array); - } - - private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) { - int n = set.size(); - int times = 10 * 1000 * 1000 / n; // run 10 million lookups - int keep = 0; // make sure JIT don't OPT OUT function calls under test - byte[] valueBytes = new byte[dict.getSizeOfValue()]; - long start; - - // benchmark value==>id, via HashMap - System.out.println(msg + " HashMap lookup value==>id"); - start = System.currentTimeMillis(); - for (int i = 0; i < times; i++) { - for (int j = 0; j < n; j++) { - keep |= map.get(strArray[j]); - } - } - long timeValueToIdByMap = System.currentTimeMillis() - start; - System.out.println(timeValueToIdByMap); - - // benchmark value==>id, via Dict - System.out.println(msg + " Dictionary lookup value==>id"); - start = System.currentTimeMillis(); - for (int i = 0; i < times; i++) { - for (int j = 0; j < n; j++) { - keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length); - } - } - long timeValueToIdByDict = System.currentTimeMillis() - start; - System.out.println(timeValueToIdByDict); - - // benchmark id==>value, via Array - System.out.println(msg + " Array lookup id==>value"); - start = System.currentTimeMillis(); - for (int i = 0; i < times; i++) { - for (int j = 0; j < n; j++) { - keep |= strArray[j].length(); - } - } - long timeIdToValueByArray = System.currentTimeMillis() - start; - System.out.println(timeIdToValueByArray); - - // benchmark id==>value, via Dict - System.out.println(msg + " Dictionary lookup id==>value"); - start = System.currentTimeMillis(); - for (int i = 0; i < times; i++) { - for (int j = 0; j < n; j++) { - keep |= dict.getValueBytesFromId(j, valueBytes, 0); - } - } - long timeIdToValueByDict = System.currentTimeMillis() - start; - System.out.println(timeIdToValueByDict); - - return keep; - } - - private static void testStringDictionary(ArrayList<String> str, ArrayList<String> notFound) { - TrieDictionaryBuilder<String> b = newDictBuilder(str); - int baseId = new Random().nextInt(100); - TrieDictionary<String> dict = b.build(baseId); - - TreeSet<String> set = new TreeSet<String>(); - for (String s : str) { - set.add(s); - } - - // test serialize - dict = testSerialize(dict); - - // test basic id<==>value - Iterator<String> it = set.iterator(); - int id = baseId; - for (; it.hasNext(); id++) { - String value = it.next(); - // System.out.println("checking " + id + " <==> " + value); - - assertEquals(id, dict.getIdFromValue(value)); - assertEquals(value, dict.getValueFromId(id)); - } - if (notFound != null) { - for (String s : notFound) { - try { - dict.getIdFromValue(s); - fail("For not found value '" + s + "', IllegalArgumentException is expected"); - } catch (IllegalArgumentException e) { - // good - } - } - } - - // test null value - int nullId = dict.getIdFromValue(null); - assertNull(dict.getValueFromId(nullId)); - int nullId2 = dict.getIdFromValueBytes(null, 0, 0); - assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1); - assertEquals(nullId, nullId2); - } - - private static TrieDictionary<String> testSerialize(TrieDictionary<String> dict) { - try { - ByteArrayOutputStream bout = new ByteArrayOutputStream(); - DataOutputStream dataout = new DataOutputStream(bout); - dict.write(dataout); - dataout.close(); - ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray()); - DataInputStream datain = new DataInputStream(bin); - TrieDictionary<String> r = new TrieDictionary<String>(); - r.readFields(datain); - datain.close(); - return r; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static TrieDictionaryBuilder<String> newDictBuilder(ArrayList<String> str) { - TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); - for (String s : str) - b.addValue(s); - return b; - } - - private static ArrayList<String> loadStrings(InputStream is) throws Exception { - ArrayList<String> r = new ArrayList<String>(); - BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); - try { - String word; - while ((word = reader.readLine()) != null) { - word = word.trim(); - if (word.isEmpty() == false) - r.add(word); - } - } finally { - reader.close(); - is.close(); - } - return r; - } - - @Test - public void testSuperLongStringValue() { - String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"; - - TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); - String v1 = longPrefix + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; - String v2 = longPrefix + "xyz"; - - b.addValue(v1); - b.addValue(v2); - TrieDictionary<String> dict = b.build(0); - dict.dump(System.out); - } - - @Test - public void testRounding() { - // see NumberDictionaryTest.testRounding(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java ---------------------------------------------------------------------- diff --git a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java deleted file mode 100644 index 3d7267d..0000000 --- a/dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict.lookup; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.TableDesc; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - * Created by shaoshi on 2/3/15. - */ -public class LookupTableTest extends LocalFileMetadataTestCase { - - private KylinConfig config = null; - - private LookupTable lookupTable; - - @Before - public void setup() throws Exception { - createTestMetadata(); - config = KylinConfig.getInstanceFromEnv(); - lookupTable = initLookupTable(); - } - - @After - public void tearDown() { - cleanupTestMetadata(); - } - - @Test - public void testScan() throws Exception { - List<String> values = new ArrayList<String>(); - values.add("2012-01-24"); - values.add("2012-12-30"); - List<String> results = lookupTable.scan("CAL_DT", values, "YEAR_BEG_DT"); - - Assert.assertTrue(results.size() > 0); - for (String i : results) { - System.out.println(i); - - Assert.assertEquals("2012-01-01", i); - } - } - - @Test - public void testMapRange() throws Exception { - List<String> values = new ArrayList<String>(); - values.add("2012-01-24"); - values.add("2012-12-30"); - Pair<String, String> results = lookupTable.mapRange("CAL_DT", "2012-01-24", "2012-12-30", "QTR_BEG_DT"); - - Assert.assertTrue(results != null); - System.out.println("The first qtr_beg_dt is " + results.getFirst()); - System.out.println("The last qtr_beg_dt is " + results.getSecond()); - - Assert.assertEquals("2012-01-01", results.getFirst()); - Assert.assertEquals("2012-10-01", results.getSecond()); - } - - @Test - public void testMapValues() throws Exception { - Set<String> values = new HashSet<String>(); - values.add("2012-01-24"); - values.add("2012-12-30"); - Set<String> results = lookupTable.mapValues("CAL_DT", values, "YEAR_BEG_DT"); - - Assert.assertTrue(results.size() == 1); - for (String i : results) { - System.out.println(i); - - Assert.assertEquals("2012-01-01", i); - } - } - - public LookupTable initLookupTable() throws Exception { - - MetadataManager metaMgr = MetadataManager.getInstance(config); - - String tableName = "EDW.TEST_CAL_DT"; - String[] pkCols = new String[] { "CAL_DT" }; - String snapshotResPath = "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot"; - SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); - TableDesc tableDesc = metaMgr.getTableDesc(tableName); - LookupTable lt = new LookupStringTable(tableDesc, pkCols, snapshot); - - System.out.println(lt); - - return lt; - } - - private SnapshotManager getSnapshotManager() { - return SnapshotManager.getInstance(config); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/dictionary/src/test/resources/dict/DW_SITES ---------------------------------------------------------------------- diff --git a/dictionary/src/test/resources/dict/DW_SITES b/dictionary/src/test/resources/dict/DW_SITES deleted file mode 100644 index 6f6af46..0000000 Binary files a/dictionary/src/test/resources/dict/DW_SITES and /dev/null differ
