Repository: kylin Updated Branches: refs/heads/2.0-rc 0de917283 -> 6ec2adbbe
KYLIN-1239 attribute each htable with team contact and owner name Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ec2adbb Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ec2adbb Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ec2adbb Branch: refs/heads/2.0-rc Commit: 6ec2adbbe3f26fb78c18b26f61a9c7f587ed34c0 Parents: 0de9172 Author: honma <ho...@ebay.com> Authored: Thu Dec 17 11:31:55 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Tue Dec 22 16:10:36 2015 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 5 +++++ .../org/apache/kylin/common/KylinConfig.java | 6 ++++++ .../realization/IRealizationConstants.java | 5 ++++- .../apache/kylin/engine/spark/SparkCubing.java | 3 +-- .../test_case_data/localmeta/kylin.properties | 5 +++++ examples/test_case_data/sandbox/kylin.properties | 5 +++++ .../storage/hbase/steps/CreateHTableJob.java | 4 +--- .../storage/hbase/steps/CubeHTableUtil.java | 19 ++++++++++++++++--- .../hbase/steps/HBaseStreamingOutput.java | 2 +- .../kylin/storage/hbase/util/CleanHtableCLI.java | 6 +++++- 10 files changed, 49 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 2c34e92..f2170bd 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -17,6 +17,11 @@ ## Config for Kylin Engine ## + +# optional information for the owner of kylin platform, it can be your team's email +# currently it will be attached to each kylin's htable attribute +kylin.owner=who...@kylin.apache.org + # List of web servers in use, this enables one web server instance to sync up with other servers. kylin.rest.servers=localhost:7070 http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index eb76534..c65ade4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -55,6 +55,8 @@ public class KylinConfig implements Serializable { * NOTE: These key constants should be private or even better be removed. * All external access should go through public methods. */ + public static final String KYLIN_OWNER = "kylin.owner"; + public static final String KYLIN_STORAGE_URL = "kylin.storage.url"; public static final String KYLIN_METADATA_URL = "kylin.metadata.url"; @@ -706,6 +708,10 @@ public class KylinConfig implements Serializable { return this.getOptional(HIVE_DATABASE_FOR_INTERMEDIATE_TABLE, "default"); } + public String getKylinOwner() { + return this.getOptional(KYLIN_OWNER, ""); + } + public void setRunAsRemoteCommand(String v) { properties.setProperty(KYLIN_JOB_RUN_AS_REMOTE_CMD, v); } http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java index 7cbcdf8..138a209 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java @@ -31,7 +31,10 @@ public class IRealizationConstants { * which kylin server(represented by its kylin_metadata prefix) owns this htable */ public final static String HTableTag = "KYLIN_HOST"; - + + public final static String HTableOwner = "OWNER"; + + public final static String HTableUser = "USER"; public final static String HTableCreationTime = "CREATION_TIME"; http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java index 01d97fd..9a2610c 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java @@ -452,12 +452,11 @@ public class SparkCubing extends AbstractApplication { private byte[][] createHTable(String cubeName, String segmentId, Map<Long, HyperLogLogPlusCounter> samplingResult) throws Exception { final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName); - final CubeDesc cubeDesc = cubeInstance.getDescriptor(); final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId); final Map<Long, Long> cubeSizeMap = CreateHTableJob.getCubeRowCountMapFromCuboidStatistics(samplingResult, 100); System.out.println("cube size estimation:" + cubeSizeMap); final byte[][] splitKeys = CreateHTableJob.getSplitsFromCuboidStatistics(cubeSizeMap, kylinConfig, cubeSegment); - CubeHTableUtil.createHTable(cubeDesc, cubeSegment.getStorageLocationIdentifier(), splitKeys); + CubeHTableUtil.createHTable(cubeSegment, splitKeys); System.out.println(cubeSegment.getStorageLocationIdentifier() + " table created"); return splitKeys; } http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/examples/test_case_data/localmeta/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index 08b351a..84c53bb 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -17,6 +17,11 @@ ## Config for Kylin Engine ## + +# optional information for the owner of kylin platform, it can be your team's email +# currently it will be attached to each kylin's htable attribute +kylin.owner=who...@kylin.apache.org + # List of web servers in use, this enables one web server instance to sync up with other servers. #kylin.rest.servers=localhost:7070 http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index d5446b5..4ca8c2d 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -17,6 +17,10 @@ ## Config for Kylin Engine ## +# optional information for the owner of kylin platform, it can be your team's email +# currently it will be attached to each kylin's htable attribute +kylin.owner=who...@kylin.apache.org + # List of web servers in use, this enables one web server instance to sync up with other servers. kylin.rest.servers=localhost:7070 @@ -26,6 +30,7 @@ kylin.metadata.url=kylin_metadata@hbase # The storage for final cube file in hbase kylin.storage.url=hbase + # Temp folder in hdfs, make sure user has the right access to the hdfs directory kylin.hdfs.working.dir=/kylin http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java index 62554b2..dfa5e90 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java @@ -91,7 +91,6 @@ public class CreateHTableJob extends AbstractHadoopJob { options.addOption(OPTION_CUBE_NAME); options.addOption(OPTION_SEGMENT_NAME); options.addOption(OPTION_PARTITION_FILE_PATH); - options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_STATISTICS_ENABLED); parseOptions(options, args); @@ -106,7 +105,6 @@ public class CreateHTableJob extends AbstractHadoopJob { segmentName = getOptionValue(OPTION_SEGMENT_NAME); CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW); - String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase(); Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); try { @@ -118,7 +116,7 @@ public class CreateHTableJob extends AbstractHadoopJob { splitKeys = getSplits(conf, partitionFilePath); } - CubeHTableUtil.createHTable(cubeDesc, tableName, splitKeys); + CubeHTableUtil.createHTable(cubeSegment, splitKeys); return 0; } catch (Exception e) { printUsage(options); http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java index cdc259b..61bea36 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java @@ -2,6 +2,7 @@ package org.apache.kylin.storage.hbase.steps; import java.io.IOException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -10,10 +11,11 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; -import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.security.User; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.HBaseColumnFamilyDesc; import org.apache.kylin.metadata.realization.IRealizationConstants; @@ -29,15 +31,26 @@ public class CubeHTableUtil { private static final Logger logger = LoggerFactory.getLogger(CubeHTableUtil.class); - public static void createHTable(CubeDesc cubeDesc, String tableName, byte[][] splitKeys) throws IOException { + public static void createHTable(CubeSegment cubeSegment, byte[][] splitKeys) throws IOException { + String tableName = cubeSegment.getStorageLocationIdentifier(); + CubeInstance cubeInstance = cubeSegment.getCubeInstance(); + CubeDesc cubeDesc = cubeInstance.getDescriptor(); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName)); + HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(cubeSegment.getStorageLocationIdentifier())); // https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.html tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); tableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); tableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis())); + if (!StringUtils.isEmpty(kylinConfig.getKylinOwner())) { + //HTableOwner is the team that provides kylin service + tableDesc.setValue(IRealizationConstants.HTableOwner, kylinConfig.getKylinOwner()); + } + + //HTableUser is the cube owner, which will be the "user" + tableDesc.setValue(IRealizationConstants.HTableUser, cubeInstance.getOwner()); + Configuration conf = HBaseConfiguration.create(); HBaseAdmin admin = new HBaseAdmin(conf); http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java index 896bc72..fdab8eb 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java @@ -83,7 +83,7 @@ public class HBaseStreamingOutput implements IStreamingOutput { private HTableInterface createHTable(final CubeSegment cubeSegment) throws IOException { final String hTableName = cubeSegment.getStorageLocationIdentifier(); - CubeHTableUtil.createHTable(cubeSegment.getCubeDesc(), hTableName, null); + CubeHTableUtil.createHTable(cubeSegment, null); final HTableInterface hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName); logger.info("hTable:" + hTableName + " for segment:" + cubeSegment.getName() + " created!"); return hTable; http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java index c583f90..957e04b 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; +import org.apache.kylin.metadata.realization.IRealizationConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,10 +58,13 @@ public class CleanHtableCLI extends AbstractHadoopJob { for (HTableDescriptor descriptor : hbaseAdmin.listTables()) { String name = descriptor.getNameAsString().toLowerCase(); if (name.startsWith("kylin") || name.startsWith("_kylin")) { - String x = descriptor.getValue("KYLIN_HOST"); + String x = descriptor.getValue(IRealizationConstants.HTableTag); System.out.println("table name " + descriptor.getNameAsString() + " host: " + x); System.out.println(descriptor); System.out.println(); + + descriptor.setValue(IRealizationConstants.HTableOwner, "dl-ebay-ky...@ebay.com"); + hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor); } } hbaseAdmin.close();
