Repository: kylin Updated Branches: refs/heads/master 8de31563c -> 9bd421fe9
enable isShardBy and int encoding for sample cubes Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9bd421fe Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9bd421fe Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9bd421fe Branch: refs/heads/master Commit: 9bd421fe92d1c4be780485c482f5cbab66f3c2c2 Parents: 8de3156 Author: Hongbin Ma <mahong...@apache.org> Authored: Wed Mar 30 11:04:47 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Wed Mar 30 11:04:47 2016 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/cube/CubeSegment.java | 4 ++-- .../java/org/apache/kylin/cube/kv/RowKeyEncoder.java | 6 +++--- .../java/org/apache/kylin/cube/model/CubeDesc.java | 4 ++-- .../org/apache/kylin/cube/model/RowKeyColDesc.java | 14 ++++++-------- .../java/org/apache/kylin/cube/model/RowKeyDesc.java | 13 ++++++------- .../cube_desc/test_kylin_cube_with_slr_desc.json | 3 ++- .../test_kylin_cube_with_slr_left_join_desc.json | 2 +- 7 files changed, 22 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd421fe/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java index 8dd294a..4fa6100 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java @@ -383,8 +383,8 @@ public class CubeSegment implements Comparable<CubeSegment>, IRealizationSegment return getCubeDesc().isEnableSharding(); } - public Set<TblColRef> getUHCColumns() { - return getCubeDesc().getUHCColumns(); + public Set<TblColRef> getShardByColumns() { + return getCubeDesc().getShardByColumns(); } public int getRowKeyPreambleSize() { http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd421fe/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java index 656ddc1..db704dc 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java @@ -48,13 +48,13 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder { public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) { super(cubeSeg, cuboid); enableSharding = cubeSeg.isEnableSharding(); - Set<TblColRef> UHCColumns = cubeSeg.getUHCColumns(); - if (UHCColumns.size() > 1) { + Set<TblColRef> shardByColumns = cubeSeg.getShardByColumns(); + if (shardByColumns.size() > 1) { throw new IllegalStateException("Does not support multiple UHC now"); } colIO = new RowKeyColumnIO(cubeSeg.getDimensionEncodingMap()); for (TblColRef column : cuboid.getColumns()) { - if (UHCColumns.contains(column)) { + if (shardByColumns.contains(column)) { UHCOffset = bodyLength; UHCLength = colIO.getColumnLength(column); } http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd421fe/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 165ab60..4abd889 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -166,8 +166,8 @@ public class CubeDesc extends RootPersistentEntity { return storageType == IStorageAware.ID_SHARDED_HBASE; } - public Set<TblColRef> getUHCColumns() { - return getRowkey().getUHCColumns(); + public Set<TblColRef> getShardByColumns() { + return getRowkey().getShardByColumns(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd421fe/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java index 08b1813..dd7d61f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java @@ -41,8 +41,8 @@ public class RowKeyColDesc { private String column; @JsonProperty("encoding") private String encoding; - @JsonProperty("isUHC") - private boolean isUHC;//is ultra high cardinality column + @JsonProperty("isShardBy") + private boolean isShardBy;//usually it is ultra high cardinality column, shard by such column can reduce the agg cache for each shard // computed private String encodingName; @@ -81,16 +81,14 @@ public class RowKeyColDesc { this.column = column; } - public boolean isUHC() { - return isUHC; + public boolean isShardBy() { + return isShardBy; } - public void setUHC(boolean UHC) { - isUHC = UHC; + public void setShardBy(boolean shardBy) { + isShardBy = shardBy; } - - public String getEncodingName() { return encodingName; } http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd421fe/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java index 2df1764..c3751df 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java @@ -43,7 +43,7 @@ public class RowKeyDesc { private long fullMask; private CubeDesc cubeDesc; private Map<TblColRef, RowKeyColDesc> columnMap; - private Set<TblColRef> UHCColumns; + private Set<TblColRef> shardByColumns; public RowKeyColDesc[] getRowKeyColumns() { return rowkeyColumns; @@ -68,8 +68,8 @@ public class RowKeyDesc { return getColDesc(col).isUsingDictionary(); } - public Set<TblColRef> getUHCColumns() { - return UHCColumns; + public Set<TblColRef> getShardByColumns() { + return shardByColumns; } public void init(CubeDesc cubeDesc) { @@ -91,7 +91,7 @@ public class RowKeyDesc { private void buildRowKey(Map<String, TblColRef> colNameAbbr) { columnMap = new HashMap<TblColRef, RowKeyColDesc>(); - UHCColumns = new HashSet<>(); + shardByColumns = new HashSet<>(); for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; @@ -106,9 +106,8 @@ public class RowKeyDesc { columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); - if (rowKeyColDesc.isUHC()) { - UHCColumns.add(rowKeyColDesc.getColRef()); - + if (rowKeyColDesc.isShardBy()) { + shardByColumns.add(rowKeyColDesc.getColRef()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd421fe/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json index 536f119..2253c60 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json @@ -113,7 +113,8 @@ "rowkey" : { "rowkey_columns" : [ { "column" : "seller_id", - "encoding" : "fixed_length:18" + "encoding" : "fixed_length:18", + "isShardBy" : true }, { "column" : "cal_dt", "encoding" : "dict" http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd421fe/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json index c0e39e7..bb59a05 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json @@ -113,7 +113,7 @@ "rowkey" : { "rowkey_columns" : [ { "column" : "seller_id", - "encoding" : "fixed_length:18" + "encoding" : "int:4" }, { "column" : "cal_dt", "encoding" : "dict"
