Repository: incubator-kylin Updated Branches: refs/heads/1.x-staging 882ca61bc -> a441237af
KYLIN-1099 Remove dedup in dictionary value enumerator Signed-off-by: Li, Yang <yang...@ebay.com> Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a441237a Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a441237a Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a441237a Branch: refs/heads/1.x-staging Commit: a441237afbf45736eea74cf6638e34de60a82cea Parents: 882ca61 Author: lidongsjtu <don...@ebay.com> Authored: Mon Nov 2 09:55:05 2015 +0800 Committer: Li, Yang <yang...@ebay.com> Committed: Wed Nov 11 10:18:13 2015 +0800 ---------------------------------------------------------------------- .../dict/ListDictionaryValueEnumerator.java | 50 ++++++++++++++++++++ .../dict/MultipleDictionaryValueEnumerator.java | 10 +--- .../kylin/dict/TableColumnValueEnumerator.java | 13 ++--- .../lookup/ListDictionaryValueEnumerator.java | 50 -------------------- .../job/hadoop/cube/MergeCuboidMapperTest.java | 2 +- 5 files changed, 55 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java new file mode 100644 index 0000000..9ae08a8 --- /dev/null +++ b/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dict; + +import org.apache.kylin.dict.IDictionaryValueEnumerator; + +import java.io.IOException; +import java.util.List; +import java.util.ListIterator; + +/** + * Created by dongli on 10/28/15. + */ +public class ListDictionaryValueEnumerator implements IDictionaryValueEnumerator { + ListIterator<byte[]> listIterator; + + public ListDictionaryValueEnumerator(List<byte[]> list) { + listIterator = list.listIterator(); + } + + @Override + public byte[] current() throws IOException { + return listIterator.next(); + } + + @Override + public boolean moveNext() throws IOException { + return listIterator.hasNext(); + } + + @Override + public void close() throws IOException { + } +} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java index 4cf72d3..13f7394 100644 --- a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java +++ b/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java @@ -19,12 +19,9 @@ package org.apache.kylin.dict; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - import org.apache.kylin.common.util.Bytes; import java.io.IOException; -import java.util.HashSet; import java.util.List; /** @@ -32,7 +29,6 @@ import java.util.List; */ @SuppressWarnings("rawtypes") public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumerator { - private HashSet<byte[]> dedup = Sets.newHashSet(); private int curDictIndex = 0; private Dictionary curDict; private int curKey; @@ -69,11 +65,7 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer } } - if (dedup.contains(curValue)) { - return moveNext(); - } else { - return true; - } + return true; } curValue = null; return false; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java index 64a0e39..42f5791 100644 --- a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java +++ b/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java @@ -18,13 +18,11 @@ package org.apache.kylin.dict; -import com.google.common.collect.Sets; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.dict.lookup.ReadableTable; import java.io.IOException; import java.util.Arrays; -import java.util.Set; /** * Created by dongli on 10/28/15. @@ -34,7 +32,6 @@ public class TableColumnValueEnumerator implements IDictionaryValueEnumerator { private ReadableTable.TableReader reader; private int colIndex; private byte[] colValue; - private Set<String> dedup = Sets.newHashSet(); public TableColumnValueEnumerator(ReadableTable.TableReader reader, int colIndex) { this.reader = reader; @@ -55,13 +52,9 @@ public class TableColumnValueEnumerator implements IDictionaryValueEnumerator { } colStrValue = split[colIndex]; } - if (!dedup.contains(colStrValue)) { - dedup.add(colStrValue); - colValue = Bytes.toBytes(colStrValue); - return true; - } else { - return moveNext(); - } + + colValue = Bytes.toBytes(colStrValue); + return true; } else { colValue = null; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java deleted file mode 100644 index 581ee43..0000000 --- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict.lookup; - -import org.apache.kylin.dict.IDictionaryValueEnumerator; - -import java.io.IOException; -import java.util.List; -import java.util.ListIterator; - -/** - * Created by dongli on 10/28/15. - */ -public class ListDictionaryValueEnumerator implements IDictionaryValueEnumerator { - ListIterator<byte[]> listIterator; - - public ListDictionaryValueEnumerator(List<byte[]> list) { - listIterator = list.listIterator(); - } - - @Override - public byte[] current() throws IOException { - return listIterator.next(); - } - - @Override - public boolean moveNext() throws IOException { - return listIterator.hasNext(); - } - - @Override - public void close() throws IOException { - } -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java index 7a2ebad..9a1fdfb 100644 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java +++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java @@ -38,7 +38,7 @@ import org.apache.kylin.dict.DictionaryGenerator; import org.apache.kylin.dict.DictionaryInfo; import org.apache.kylin.dict.DictionaryManager; import org.apache.kylin.dict.TrieDictionary; -import org.apache.kylin.dict.lookup.ListDictionaryValueEnumerator; +import org.apache.kylin.dict.ListDictionaryValueEnumerator; import org.apache.kylin.dict.lookup.ReadableTable.TableSignature; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.TblColRef;
