KYLIN_2752 code clean up
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/876811f2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/876811f2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/876811f2 Branch: refs/heads/KYLIN-2846 Commit: 876811f2312981b99344cd700e1300a0462ec124 Parents: c9a855f Author: Li Yang <liy...@apache.org> Authored: Sun Sep 17 16:21:12 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sun Sep 17 16:28:19 2017 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/cube/CubeManagerTest.java | 12 ++++++++---- .../metadata/realization/IRealizationConstants.java | 4 ---- examples/test_case_data/localmeta/kylin.properties | 6 ------ .../apache/kylin/rest/job/StorageCleanJobHbaseUtil.java | 7 ++++--- .../org/apache/kylin/storage/hbase/util/HBaseClean.java | 5 +++-- .../org/apache/kylin/storage/hbase/util/HBaseUsage.java | 5 +++-- .../kylin/storage/hbase/util/OrphanHBaseCleanJob.java | 5 +++-- .../kylin/storage/hbase/util/StorageCleanupJob.java | 9 +++++---- 8 files changed, 26 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index da27032..0188904 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -307,10 +307,14 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { @Test public void testGetCubeNameWithNamespace() { - CubeManager mgr = CubeManager.getInstance(getTestConfig()); - String tablename = mgr.generateStorageLocation(); - System.out.println(tablename); - assertTrue(tablename.startsWith("HELLO:WORLD")); + System.setProperty("kylin.storage.hbase.table-name-prefix", "HELLO_"); + try { + CubeManager mgr = CubeManager.getInstance(getTestConfig()); + String tablename = mgr.generateStorageLocation(); + assertTrue(tablename.startsWith("HELLO_")); + } finally { + System.clearProperty("kylin.storage.hbase.table-name-prefix"); + } } public CubeDescManager getCubeDescManager() { http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java index 2a3b01e..2e4f400 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java @@ -22,10 +22,6 @@ package org.apache.kylin.metadata.realization; */ public class IRealizationConstants { - public final static String SharedHbaseStorageLocationPrefix = "KYLIN_"; - public final static String CubeHbaseStorageLocationPrefix = "KYLIN_"; - public final static String IIHbaseStorageLocationPrefix = "KYLIN_II_"; - /** * For each cube htable, we leverage htable's metadata to keep track of * which kylin server(represented by its kylin_metadata prefix) owns this htable http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/examples/test_case_data/localmeta/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index 48e2856..c7dda3f 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -39,12 +39,6 @@ kylin.metadata.url= # The storage for final cube file in hbase kylin.storage.url=hbase -# The prefix for cube table -kylin.storage.hbase.tableprefix=WORLD_ - -# The namespace for hbase storage -kylin.storage.hbase.namespace=HELLO - # Working folder in HDFS, make sure user has the right access to the hdfs directory kylin.env.hdfs-working-dir=/kylin http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java index 3728ea1..aceebb4 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java +++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java @@ -46,15 +46,16 @@ public class StorageCleanJobHbaseUtil { public static void cleanUnusedHBaseTables(boolean delete, int deleteTimeout) throws IOException { Configuration conf = HBaseConfiguration.create(); - CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + CubeManager cubeMgr = CubeManager.getInstance(kylinConfig); // get all kylin hbase tables try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) { - String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; + String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix(); HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); List<String> allTablesNeedToBeDropped = new ArrayList<String>(); for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); - if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) { + if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) { //only take care htables that belongs to self, and created more than 2 days allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java index 940d64a..1bafeb2 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java @@ -57,9 +57,10 @@ public class HBaseClean extends AbstractApplication { private void cleanUp() { try { // get all kylin hbase tables - Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); - String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; + String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix(); HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); List<String> allTablesNeedToBeDropped = Lists.newArrayList(); for (HTableDescriptor desc : tableDescriptors) { http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java index a2f60d4..bb59242 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java @@ -43,9 +43,10 @@ public class HBaseUsage { Map<String, List<String>> envs = Maps.newHashMap(); // get all kylin hbase tables - Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); - String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; + String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix(); HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java index df4e912..0c35e46 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java @@ -56,10 +56,11 @@ public class OrphanHBaseCleanJob extends AbstractApplication { Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); private void cleanUnusedHBaseTables(Configuration conf) throws IOException { - Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); // get all kylin hbase tables Admin hbaseAdmin = conn.getAdmin(); - String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; + String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix(); HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); List<String> allTablesNeedToBeDropped = new ArrayList<String>(); for (HTableDescriptor desc : tableDescriptors) { http://git-wip-us.apache.org/repos/asf/kylin/blob/876811f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java index f6b65ab..2ae21b4 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java @@ -78,16 +78,17 @@ public class StorageCleanupJob extends AbstractApplication { protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); private void cleanUnusedHBaseTables(Configuration conf) throws IOException { - CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + CubeManager cubeMgr = CubeManager.getInstance(kylinConfig); // get all kylin hbase tables - Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); + Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); - String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; + String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix(); HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); List<String> allTablesNeedToBeDropped = new ArrayList<String>(); for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); - if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) { + if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) { //only take care htables that belongs to self, and created more than 2 days allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString()); }
