Repository: kylin Updated Branches: refs/heads/master 7976b5fc7 -> 3a3e96786
KYLIN-2477 modify zk connection url Signed-off-by: Hongbin Ma <mahong...@apache.org> remove tests Signed-off-by: Hongbin Ma <mahong...@apache.org> minor Signed-off-by: Hongbin Ma <mahong...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a3e9678 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a3e9678 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a3e9678 Branch: refs/heads/master Commit: 3a3e96786b5bbea046d0a6fe9eef1c3f8cb13521 Parents: 7976b5f Author: xiefan46 <958034...@qq.com> Authored: Sun Mar 12 13:15:38 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Mon Mar 13 09:13:16 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 9 ----- .../test_case_data/sandbox/kylin.properties | 2 - .../storage/hdfs/ITHDFSResourceStoreTest.java | 40 -------------------- .../kylin/storage/hdfs/ITLockManagerTest.java | 8 ++-- .../apache/kylin/storage/hdfs/LockManager.java | 18 +-------- 5 files changed, 5 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3e9678/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index ad02fe2..02349ad 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -940,15 +940,6 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true")); } - //zoo keeper - public String getZooKeeperHost() { - return getOptional("kylin.storage.lock-manager-zk-host", "localhost"); - } - - public String getZooKeeperPort() { - return getOptional("kylin.storage.lock-manager-zk-port", "2181"); - } - //ResourceStore Impl /*public String getResourceStoreImpl() { return getOptional("kylin.metadata.default-resource-store-impl", "org.apache.kylin.storage.hbase.HBaseResourceStore"); http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3e9678/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 7040cf3..6e360a3 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -55,8 +55,6 @@ kylin.env.hdfs-working-dir=/kylin kylin.engine.mr.reduce-input-mb=500 -#zoo keeper host in lock manager -kylin.storage.lock-manager-zk-host=sandbox ### JOB ### http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3e9678/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java index 0e21287..82e0406 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java @@ -18,15 +18,10 @@ package org.apache.kylin.storage.hdfs; -import static junit.framework.TestCase.assertTrue; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStoreTest; import org.apache.kylin.common.util.HBaseMetadataTestCase; -import org.apache.kylin.common.util.HadoopUtil; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -36,16 +31,10 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase { private KylinConfig kylinConfig; - private FileSystem fs; - - private String workingDir; - @Before public void setup() throws Exception { this.createTestMetadata(); kylinConfig = KylinConfig.getInstanceFromEnv(); - fs = HadoopUtil.getWorkingFileSystem(); - workingDir = getHdfsWorkingDirWithoutScheme(kylinConfig); } @After @@ -53,46 +42,17 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase { this.cleanupTestMetadata(); } - private String getHdfsWorkingDirWithoutScheme(KylinConfig kylinConfig) { - String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory(); - int thirdIndex = StringUtils.ordinalIndexOf(hdfsWorkingDir, "/", 3); - int fourthIndex = StringUtils.ordinalIndexOf(hdfsWorkingDir, "/", 5); - return hdfsWorkingDir.substring(thirdIndex, fourthIndex); - } - @Test public void testBasic() throws Exception { - String path = workingDir + "/metadata_test1"; - doTestWithPath(path); - } - - @Test - public void testQalifiedName() throws Exception { - String path = "hdfs://" + workingDir + "/metadata_test2"; - doTestWithPath(path); - } - - @Test - public void testFullQalifiedName() throws Exception { - String path = fs.getUri() + workingDir + "/metadata_test3"; - doTestWithPath(path); - } - - private void doTestWithPath(String path) throws Exception { ResourceStoreTest.testAStore(ResourceStoreTest.mockUrl("hdfs", kylinConfig), kylinConfig); - assertTrue(fs.exists(new Path(path))); } @Ignore @Test public void performanceTest() throws Exception { - //test hdfs performance - String hdfsStoreName = "org.apache.kylin.storage.hdfs.HDFSResourceStore"; ResourceStoreTest.testPerformance(ResourceStoreTest.mockUrl("hdfs", kylinConfig), kylinConfig); - //test hbase - String hbaseStoreName = "org.apache.kylin.storage.hbase.HBaseResourceStore.HBaseResourceStore"; ResourceStoreTest.testPerformance(ResourceStoreTest.mockUrl("hbase", kylinConfig), kylinConfig); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3e9678/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java index 3449f87..5587eee 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java @@ -24,6 +24,7 @@ import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.utils.CloseableUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HBaseMetadataTestCase; +import org.apache.kylin.storage.hbase.util.ZookeeperUtil; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -45,9 +46,6 @@ import static org.junit.Assert.assertEquals; public class ITLockManagerTest extends HBaseMetadataTestCase { - - private String zkConnection = "sandbox:2181"; - private KylinConfig kylinConfig; private CuratorFramework zkClient; @@ -67,7 +65,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase { this.createTestMetadata(); kylinConfig = KylinConfig.getInstanceFromEnv(); RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); - zkClient = CuratorFrameworkFactory.newClient(zkConnection, retryPolicy); + zkClient = CuratorFrameworkFactory.newClient(ZookeeperUtil.getZKConnectString(), retryPolicy); zkClient.start(); manager = new LockManager(kylinConfig, lockRootPath); logger.info("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath)); @@ -102,7 +100,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase { // FakeLimitedResource simulates some external resource that can only be access by one process at a time final FakeLimitedResource resource = new FakeLimitedResource(); ExecutorService service = Executors.newFixedThreadPool(QTY); - final TestingServer server = new TestingServer(zkConnection); + final TestingServer server = new TestingServer(ZookeeperUtil.getZKConnectString()); final List<FutureTask<Void>> tasks = new ArrayList<>(); try { for (int i = 0; i < QTY; ++i) { http://git-wip-us.apache.org/repos/asf/kylin/blob/3a3e9678/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java index 582482f..b04ea74 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java @@ -17,8 +17,6 @@ */ package org.apache.kylin.storage.hdfs; -import com.google.common.base.Function; -import com.google.common.collect.Iterables; import org.apache.commons.lang3.StringUtils; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; @@ -27,12 +25,11 @@ import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.storage.hbase.util.ZookeeperDistributedJobLock; +import org.apache.kylin.storage.hbase.util.ZookeeperUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; import java.io.IOException; -import java.util.Arrays; public class LockManager { @@ -52,7 +49,7 @@ public class LockManager { public LockManager(KylinConfig config, String lockRootPath) throws Exception { this.config = config; this.lockRootPath = lockRootPath; - String zkConnectString = getZKConnectString(config); + String zkConnectString = ZookeeperUtil.getZKConnectString(); logger.info("zk connection string:" + zkConnectString); if (StringUtils.isEmpty(zkConnectString)) { throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!"); @@ -86,17 +83,6 @@ public class LockManager { } - private static String getZKConnectString(KylinConfig kylinConfig) { - final String host = kylinConfig.getZooKeeperHost(); - final String port = kylinConfig.getZooKeeperPort(); - return StringUtils.join(Iterables.transform(Arrays.asList(host.split(",")), new Function<String, String>() { - @Nullable - @Override - public String apply(String input) { - return input + ":" + port; - } - }), ","); - } public String getLockPath(String resourceName) { if (!resourceName.startsWith("/"))
