KYLIN-2790 Add extending point for column family
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d62cb58b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d62cb58b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d62cb58b Branch: refs/heads/master Commit: d62cb58bde55766dbe87bbad29953dd1243c2014 Parents: 4782289 Author: auphyroc99 <454530...@qq.com> Authored: Wed Aug 16 10:34:02 2017 +0800 Committer: Roger Shi <rogershijich...@gmail.com> Committed: Wed Aug 30 18:13:52 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 4 ++++ .../org/apache/kylin/cube/model/CubeDesc.java | 18 ++++++++++++++-- .../kylin/cube/model/HBaseMappingDesc.java | 18 ++++++++++++++++ .../org/apache/kylin/gridtable/GTRecord.java | 22 ++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index a113327..47806a5 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -286,6 +286,10 @@ abstract public class KylinConfigBase implements Serializable { "org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory"); return (DistributedLockFactory) ClassUtil.newInstance(clsName); } + + public String getHBaseMappingAdapter() { + return getOptional("kylin.metadata.hbasemapping-adapter"); + } // ============================================================================ // DICTIONARY & SNAPSHOT http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 0e22587..d263e1f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import java.lang.reflect.Method; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -603,8 +604,21 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { validateAggregationGroups(); // check if aggregation group is valid validateAggregationGroupsCombination(); - if (hbaseMapping != null) { - hbaseMapping.init(this); + String hbaseMappingAdapterName = config.getHBaseMappingAdapter(); + + if (hbaseMappingAdapterName != null) { + try { + Class<?> hbaseMappingAdapterClass = Class.forName(hbaseMappingAdapterName); + Method initMethod = hbaseMappingAdapterClass.getMethod("initHBaseMapping", CubeDesc.class); + initMethod.invoke(null, this); + } catch (Exception e) { + logger.error("Wrong configuration for kylin.metadata.hbasemapping-adapter: class " + + hbaseMappingAdapterName + " not found. "); + } + } else { + if (hbaseMapping != null) { + hbaseMapping.init(this); + } } initMeasureReferenceToColumnFamily(); http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java index d1e5829..77b4b14 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java @@ -87,6 +87,24 @@ public class HBaseMappingDesc implements java.io.Serializable { } } + public void initAsSeparatedColumns(CubeDesc cubeDesc) { + cubeRef = cubeDesc; + + int cfNum = cubeDesc.getMeasures().size(); + columnFamily = new HBaseColumnFamilyDesc[cfNum]; + + for (int i = 0; i < cfNum; i++) { + HBaseColumnFamilyDesc cf = new HBaseColumnFamilyDesc(); + HBaseColumnDesc col = new HBaseColumnDesc(); + String measureRef = cubeDesc.getMeasures().get(i).getName(); + col.setMeasureRefs(new String[] { measureRef }); + col.setQualifier("M"); + cf.setColumns(new HBaseColumnDesc[] { col }); + cf.setName("F" + (i + 1)); + columnFamily[i] = cf; + } + } + @Override public String toString() { return "HBaseMappingDesc [columnFamily=" + Arrays.toString(columnFamily) + "]"; http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java index 3e62ea7..862fb99 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java @@ -295,5 +295,27 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable { buf.position(pos); } } + + public void loadColumnsFromColumnBlocks(ImmutableBitSet[] selectedColumnBlocks, ImmutableBitSet selectedCols, ByteBuffer buf) { + + int pos = buf.position(); + + for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) { + + for (int i = 0; i < selectedColBlock.trueBitCount(); i++) { + + int c = selectedColBlock.trueBitAt(i); + + int len = info.codeSystem.codeLength(c, buf); + if(selectedCols.get(c)) { + cols[c].set(buf.array(), buf.arrayOffset() + pos, len); + } + pos += len; + buf.position(pos); + } + + } + + } }
