Repository: incubator-kylin Updated Branches: refs/heads/2.x-staging 8a77b8600 -> b3fefd140
KYLIN-1116 use local dictionary for inverted index building Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b3fefd14 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b3fefd14 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b3fefd14 Branch: refs/heads/2.x-staging Commit: b3fefd1406b81c87e3cdede96c010fcade7ef1a0 Parents: d5d8a40 Author: shaofengshi <shaofeng...@apache.org> Authored: Fri Nov 6 20:02:30 2015 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Fri Nov 6 20:04:02 2015 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/job/BuildIIWithStreamTest.java | 2 +- .../org/apache/kylin/job/hadoop/invertedindex/IITest.java | 2 +- .../engine/mr/invertedindex/InvertedIndexReducer.java | 2 +- .../apache/kylin/invertedindex/index/SliceBuilder.java | 6 ++---- .../java/org/apache/kylin/invertedindex/model/IIDesc.java | 10 ---------- 5 files changed, 5 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b3fefd14/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java index 2d40b09..ddfd399 100644 --- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java +++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java @@ -190,7 +190,7 @@ public class BuildIIWithStreamTest { ToolRunner.run(new IICreateHTableJob(), args); final IIDesc iiDesc = segment.getIIDesc(); - final SliceBuilder sliceBuilder = new SliceBuilder(desc, (short) 0, iiDesc.isUseLocalDictionary()); + final SliceBuilder sliceBuilder = new SliceBuilder(desc, (short) 0); List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn()); int count = sorted.size(); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b3fefd14/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java index a393ce3..ac91efb 100644 --- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java +++ b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java @@ -102,7 +102,7 @@ public class IITest extends LocalFileMetadataTestCase { StreamingBatch batch = new StreamingBatch(streamingMessages, Pair.newPair(0L, System.currentTimeMillis())); iiRows = Lists.newArrayList(); - final Slice slice = new SliceBuilder(iiDesc, (short) 0, true).buildSlice((batch)); + final Slice slice = new SliceBuilder(iiDesc, (short) 0).buildSlice((batch)); IIKeyValueCodec codec = new IIKeyValueCodec(slice.getInfo()); for (IIRow iiRow : codec.encodeKeyValue(slice)) { iiRows.add(iiRow); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b3fefd14/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java index 56c0b9e..c223159 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java @@ -73,7 +73,7 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, Object, Imm sliceSize = ii.getDescriptor().getSliceSize(); short shard = (short) context.getTaskAttemptID().getTaskID().getId(); System.out.println("Generating to shard - " + shard); - sliceBuilder = new SliceBuilder(seg.getIIDesc(), shard, true); + sliceBuilder = new SliceBuilder(seg.getIIDesc(), shard); messages = Lists.newArrayListWithCapacity(sliceSize); immutableBytesWritable = new ImmutableBytesWritable(); valueBuf = ByteBuffer.allocate(1024 * 1024); // 1MB http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b3fefd14/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java index 1c293d7..ef63b0f 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java @@ -38,12 +38,10 @@ public final class SliceBuilder { private final BatchSliceMaker sliceMaker; private final IIDesc iiDesc; - private final boolean useLocalDict; - public SliceBuilder(IIDesc desc, short shard, boolean useLocalDict) { + public SliceBuilder(IIDesc desc, short shard) { this.iiDesc = desc; this.sliceMaker = new BatchSliceMaker(desc, shard); - this.useLocalDict = useLocalDict; } public Slice buildSlice(StreamingBatch microStreamBatch) { @@ -54,7 +52,7 @@ public final class SliceBuilder { return input.getData(); } }); - final Dictionary<?>[] dictionaries = useLocalDict ? IIDictionaryBuilder.buildDictionary(messages, iiDesc) : new Dictionary[iiDesc.listAllColumns().size()]; + final Dictionary<?>[] dictionaries = IIDictionaryBuilder.buildDictionary(messages, iiDesc); TableRecordInfo tableRecordInfo = new TableRecordInfo(iiDesc, dictionaries); return build(messages, tableRecordInfo, dictionaries); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b3fefd14/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java index bfa4eaa..71737dc 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java @@ -78,8 +78,6 @@ public class IIDesc extends RootPersistentEntity { private short sharding = 1; // parallelism @JsonProperty("slice_size") private int sliceSize = 50000; // no. rows - @JsonProperty("useLocalDictionary") - private boolean useLocalDictionary = true; @JsonProperty("engine_type") private int engineType = IEngineAware.ID_MR_II; @@ -374,14 +372,6 @@ public class IIDesc extends RootPersistentEntity { this.name = name; } - public boolean isUseLocalDictionary() { - return useLocalDictionary; - } - - public void setUseLocalDictionary(boolean useLocalDictionary) { - this.useLocalDictionary = useLocalDictionary; - } - public String calculateSignature() { MessageDigest md = null; try {
