yiming187 closed pull request #41: add kylin properties for hadoop-azure
URL: https://github.com/apache/kylin/pull/41
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 131a725951..ddd63c07ad 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -57,6 +57,17 @@ kylin.source.hive.database-for-flat-table=default
# Whether redistribute the intermediate flat table before building
kylin.source.hive.redistribute-flat-table=true
+# Hadoop Cluster FileSystem, which serving hive and mapreduce, format as
hdfs://hadoop-cluster:8020
+# Leave empty if hive and mapreduce running on same cluster with hbase
+#kylin.source.hive.cluster-fs=
+
+# Azure storage account
+# Leave empty if kylin not running on hadoop-azure or hive and mapreduce
running on same cluster with hbase
+#kylin.source.hive.azure.account=
+
+# Azure storage access key
+# Leave empty if kylin not running on hadoop-azure or hive and mapreduce
running on same cluster with hbase
+#kylin.source.hive.azure.account-key=
### STORAGE ###
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3c108263dd..e4fccd764e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -493,6 +493,18 @@ public String getHiveBeelineParams() {
return getOptional("kylin.source.hive.beeline-params", "");
}
+ public String getHiveClusterFs() {
+ return getOptional("kylin.source.hive.cluster-fs", "");
+ }
+
+ public String getHiveAzureAccount() {
+ return getOptional("kylin.source.hive.azure.account", "");
+ }
+
+ public String getHiveAzureAccountKey() {
+ return getOptional("kylin.source.hive.azure.account-key", "");
+ }
+
@Deprecated
public String getCreateFlatHiveTableMethod() {
return getOptional("kylin.source.hive.create-flat-table-method", "1");
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 88692a08d0..b3aa800e3d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Writable;
+import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,6 +38,7 @@
@SuppressWarnings("unused")
private static final Logger logger =
LoggerFactory.getLogger(HadoopUtil.class);
private static final ThreadLocal<Configuration> hadoopConfig = new
ThreadLocal<>();
+ public static final String DFS_AZURE_ACCOUNT_KEY =
"fs.azure.account.key.<accountname>.blob.core.windows.net";
public static void setCurrentConfiguration(Configuration conf) {
hadoopConfig.set(conf);
@@ -56,6 +58,18 @@ private static Configuration healSickConfig(Configuration
conf) {
// why we have this hard code?
conf.set(DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY,
"8");
+ // support hive/mapreduce running on a different FS
+ String hiveClusterFs =
KylinConfig.getInstanceFromEnv().getHiveClusterFs();
+ if (StringUtils.isNotEmpty(hiveClusterFs)) {
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, hiveClusterFs);
+ }
+ // support hive/mapreduce running on a different azure storage
container, https://hadoop.apache.org/docs/current/hadoop-azure/index.html
+ String azureAccount =
KylinConfig.getInstanceFromEnv().getHiveAzureAccount();
+ String azureAccountKey =
KylinConfig.getInstanceFromEnv().getHiveAzureAccountKey();
+ if (StringUtils.isNotEmpty(azureAccount) &&
StringUtils.isNotEmpty(azureAccountKey)) {
+ String azureAccountKeyName =
DFS_AZURE_ACCOUNT_KEY.replace("<accountname>", azureAccount);
+ conf.set(azureAccountKeyName, azureAccountKey);
+ }
// https://issues.apache.org/jira/browse/KYLIN-953
if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) {
conf.set("hadoop.tmp.dir", "/tmp");
diff --git
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java
index 3c4195fa44..2c159ab83e 100644
---
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java
+++
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java
@@ -31,12 +31,15 @@
import java.util.Random;
import java.util.UUID;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.dict.ByteComparator;
import org.apache.kylin.dict.BytesConverter;
import org.apache.kylin.dict.IDictionaryValueEnumerator;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TableColumnValueEnumerator;
import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@@ -45,6 +48,15 @@
*/
public class SortedColumnReaderTest {
+ @Before
+ public void setUp() throws Exception {
+ LocalFileMetadataTestCase.staticCreateTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ LocalFileMetadataTestCase.cleanAfterClass();
+ }
@Test
public void testReadStringMultiFile() throws Exception {
String dirPath = "src/test/resources/multi_file_str";
diff --git
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java
index 4c43dbcc24..5d8ca532d4 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/TableReaderTest.java
@@ -24,6 +24,9 @@
import java.io.IOException;
import java.util.Arrays;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
/**
@@ -31,7 +34,15 @@
*
*/
public class TableReaderTest {
+ @Before
+ public void setUp() throws Exception {
+ LocalFileMetadataTestCase.staticCreateTestMetadata();
+ }
+ @After
+ public void after() throws Exception {
+ LocalFileMetadataTestCase.cleanAfterClass();
+ }
@Test
public void testBasicReader() throws IOException {
File f = new File("src/test/resources/dict/DW_SITES");
diff --git
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
index f99488657f..41ef580e74 100644
---
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
+++
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CreateHTableTest.java
@@ -39,13 +39,12 @@
@Before
public void setup() throws Exception {
+ this.createTestMetadata();
+
conf = HadoopUtil.getCurrentConfiguration();
conf.set("fs.default.name", "file:///");
conf.set("mapreduce.framework.name", "local");
conf.set("mapreduce.application.framework.path", "");
-
- this.createTestMetadata();
-
}
@After
diff --git
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
index 70e1ac7946..ecc44f6c20 100644
---
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
+++
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJobTest.java
@@ -37,6 +37,7 @@
@Before
public void setup() throws Exception {
+ createTestMetadata();
conf = HadoopUtil.getCurrentConfiguration();
conf.set("fs.default.name", "file:///");
conf.set("mapreduce.framework.name", "local");
@@ -44,7 +45,6 @@ public void setup() throws Exception {
// for local runner out-of-memory issue
conf.set("mapreduce.task.io.sort.mb", "10");
- createTestMetadata();
}
@After
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
