This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from a334ee4 KYLIN-3442 Fact distinct columns in Spark
new 637f45d KYLIN-3491 add a shrunken global dictionary step to improve
the encoding process
new 80c60ee KYLIN-3491 enable shrunken gd in ci test
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/kylin/common/KylinConfigBase.java | 4 +
.../java/org/apache/kylin/cube/model/CubeDesc.java | 4 +
.../org/apache/kylin/dict/ShrunkenDictionary.java | 159 +++++++++++++++++++++
...Builder.java => ShrunkenDictionaryBuilder.java} | 39 +++--
.../apache/kylin/dict/ShrunkenDictionaryTest.java | 77 ++++++++++
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 10 ++
.../java/org/apache/kylin/engine/mr/IMRInput.java | 4 +
.../apache/kylin/engine/mr/JobBuilderSupport.java | 22 +++
.../kylin/engine/mr/common/AbstractHadoopJob.java | 3 +
.../kylin/engine/mr/common/BaseCuboidBuilder.java | 6 +-
.../kylin/engine/mr/common/BatchConstants.java | 1 +
.../engine/mr/common/DictionaryGetterUtil.java | 76 ++++++++++
.../engine/mr/steps/BaseCuboidMapperBase.java | 10 +-
.../apache/kylin/engine/mr/steps/CuboidJob.java | 5 +
...ob.java => ExtractDictionaryFromGlobalJob.java} | 72 +++++-----
.../steps/ExtractDictionaryFromGlobalMapper.java | 150 +++++++++++++++++++
.../kylin/engine/mr/steps/InMemCuboidJob.java | 5 +
.../engine/mr/steps/InMemCuboidMapperBase.java | 15 +-
.../template/cube/kylin_sales_cube.json | 8 +-
.../template/cube/kylin_streaming_cube.json | 2 +-
.../localmeta/cube_desc/ci_left_join_cube.json | 3 +-
.../org/apache/kylin/source/hive/HiveMRInput.java | 9 ++
.../apache/kylin/source/kafka/KafkaMRInput.java | 7 +
.../kylin/storage/hbase/steps/HBaseJobSteps.java | 1 +
25 files changed, 620 insertions(+), 73 deletions(-)
create mode 100644
core-dictionary/src/main/java/org/apache/kylin/dict/ShrunkenDictionary.java
copy
core-dictionary/src/main/java/org/apache/kylin/dict/{IDictionaryBuilder.java =>
ShrunkenDictionaryBuilder.java} (52%)
create mode 100644
core-dictionary/src/test/java/org/apache/kylin/dict/ShrunkenDictionaryTest.java
create mode 100644
engine-mr/src/main/java/org/apache/kylin/engine/mr/common/DictionaryGetterUtil.java
copy
engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/{UpdateOldCuboidShardJob.java
=> ExtractDictionaryFromGlobalJob.java} (65%)
create mode 100644
engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ExtractDictionaryFromGlobalMapper.java
