Repository: kylin Updated Branches: refs/heads/master 185b34ff6 -> f0fe457e3
KYLIN-1706 Override MR job conf by kylin properties Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0fe457e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0fe457e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0fe457e Branch: refs/heads/master Commit: f0fe457e387235107921ee58f5a7e23a53938d02 Parents: 185b34f Author: Li Yang <[email protected]> Authored: Wed May 18 16:14:54 2016 +0800 Committer: Li Yang <[email protected]> Committed: Wed May 18 16:14:54 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 18 ++ .../apache/kylin/common/KylinConfigTest.java | 52 ++++++ .../kylin/cube/CubeSpecificConfigTest.java | 130 +++++++-------- .../engine/mr/common/AbstractHadoopJob.java | 15 +- .../test_case_data/localmeta/kylin.properties | 164 ++++++++++--------- 5 files changed, 227 insertions(+), 152 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 14dda82..f4b359f 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -22,6 +22,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; +import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.SortedSet; import java.util.regex.Matcher; @@ -33,6 +35,7 @@ import org.apache.kylin.common.util.CliCommandExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; @SuppressWarnings("serial") @@ -85,6 +88,17 @@ abstract public class KylinConfigBase implements Serializable { protected Properties getAllProperties() { return properties; } + + final protected Map<String, String> getPropertiesByPrefix(String prefix) { + Map<String, String> result = Maps.newLinkedHashMap(); + for (Entry<Object, Object> entry : getAllProperties().entrySet()) { + String key = (String) entry.getKey(); + if (key.startsWith(prefix)) { + result.put(key.substring(prefix.length()), (String) entry.getValue()); + } + } + return result; + } final protected String[] getOptionalStringArray(String prop, String[] dft) { final String property = getOptional(prop); @@ -225,6 +239,10 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.job.mr.lib.dir", ""); } + public Map<String, String> getMRConfigOverride() { + return getPropertiesByPrefix("kylin.job.mr.config.override."); + } + public String getKylinSparkJobJarPath() { final String jobJar = getOptional("kylin.job.jar.spark"); if (StringUtils.isNotEmpty(jobJar)) { http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java new file mode 100644 index 0000000..9ce1425 --- /dev/null +++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common; + +import static org.junit.Assert.*; + +import java.util.Map; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class KylinConfigTest extends LocalFileMetadataTestCase { + + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + @Test + public void testMRConfigOverride() { + KylinConfig config = KylinConfig.getInstanceFromEnv(); + Map<String, String> override = config.getMRConfigOverride(); + assertEquals(2, override.size()); + assertEquals("test1", override.get("test1")); + assertEquals("test2", override.get("test2")); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java index 00cc66c..51d4e54 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java @@ -1,70 +1,60 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.cube; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Arrays; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.validation.IValidatorRule; -import org.apache.kylin.cube.model.validation.ValidateContext; -import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class CubeSpecificConfigTest extends LocalFileMetadataTestCase { - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void test() { - KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); - CubeDesc cubeDesc = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb"); - verifyOverride(baseConfig, cubeDesc.getConfig()); - } - - @Test - public void test2() { - KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); - CubeInstance cube = CubeManager.getInstance(baseConfig).getCube("ssb"); - verifyOverride(baseConfig, cube.getConfig()); - } - - private void verifyOverride(KylinConfig base, KylinConfig override) { - assertEquals("gzip", base.getHbaseDefaultCompressionCodec()); - assertEquals("lz4", override.getHbaseDefaultCompressionCodec()); - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.cube; + +import static org.junit.Assert.*; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.cube.model.CubeDesc; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class CubeSpecificConfigTest extends LocalFileMetadataTestCase { + + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + @Test + public void test() { + KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); + CubeDesc cubeDesc = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb"); + verifyOverride(baseConfig, cubeDesc.getConfig()); + } + + @Test + public void test2() { + KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); + CubeInstance cube = CubeManager.getInstance(baseConfig).getCube("ssb"); + verifyOverride(baseConfig, cube.getConfig()); + } + + private void verifyOverride(KylinConfig base, KylinConfig override) { + assertEquals("gzip", base.getHbaseDefaultCompressionCodec()); + assertEquals("lz4", override.getHbaseDefaultCompressionCodec()); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index e4f253f..737ef0e 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -154,7 +155,9 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { } protected void setJobClasspath(Job job) { - String jarPath = KylinConfig.getInstanceFromEnv().getKylinJobJarPath(); + KylinConfig kylinConf = KylinConfig.getInstanceFromEnv(); + + String jarPath = kylinConf.getKylinJobJarPath(); File jarFile = new File(jarPath); if (jarFile.exists()) { job.setJar(jarPath); @@ -223,7 +226,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { } // for KylinJobMRLibDir - String mrLibDir = KylinConfig.getInstanceFromEnv().getKylinJobMRLibDir(); + String mrLibDir = kylinConf.getKylinJobMRLibDir(); if (!StringUtils.isBlank(mrLibDir)) { File dirFileMRLIB = new File(mrLibDir); if (dirFileMRLIB.exists()) { @@ -236,6 +239,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { } setJobTmpJarsAndFiles(job, kylinDependency.toString()); + + overrideJobConfig(job.getConfiguration(), kylinConf.getMRConfigOverride()); + } + + private void overrideJobConfig(Configuration jobConf, Map<String, String> override) { + for (Entry<String, String> entry : override.entrySet()) { + jobConf.set(entry.getKey(), entry.getValue()); + } } private String filterKylinHiveDependency(String kylinHiveDependency) { http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/examples/test_case_data/localmeta/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index 051a312..9ac7625 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -1,80 +1,84 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# optional information for the owner of kylin platform, it can be your team's email -# currently it will be attached to each kylin's htable attribute [email protected] - -# List of web servers in use, this enables one web server instance to sync up with other servers. -#kylin.rest.servers=localhost:7070 - -# The metadata store in hbase -kylin.metadata.url= - -# The storage for final cube file in hbase -kylin.storage.url=hbase - -# Temp folder in hdfs, make sure user has the right access to the hdfs directory -kylin.hdfs.working.dir=/kylin - -kylin.job.mapreduce.default.reduce.input.mb=500 - -# If true, job engine will not assume that hadoop CLI reside on the same server as it self -# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password -kylin.job.run.as.remote.cmd=false - -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.hostname= - -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.username= - -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.password= - -# Used by test cases to prepare synthetic data for sample cube -kylin.job.remote.cli.working.dir=/tmp/kylin - -# Max count of concurrent jobs running -kylin.job.concurrent.max.limit=10 - -# Time interval to check hadoop job status -kylin.job.yarn.app.rest.check.interval.seconds=10 - -#default compression codec for htable,snappy,lzo,gzip,lz4 -kylin.hbase.default.compression.codec=gzip - - -kylin.security.profile=testing -## Config for Restful APP ## -# database connection settings: -ldap.server= -ldap.username= -ldap.password= -ldap.user.searchBase= -ldap.user.searchPattern= -ldap.user.groupSearchBase= -ldap.service.searchBase=OU= -ldap.service.searchPattern= -ldap.service.groupSearchBase= -acl.adminRole= -acl.defaultRole= -ganglia.group= -ganglia.port=8664 - - - +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# optional information for the owner of kylin platform, it can be your team's email +# currently it will be attached to each kylin's htable attribute [email protected] + +# List of web servers in use, this enables one web server instance to sync up with other servers. +#kylin.rest.servers=localhost:7070 + +# The metadata store in hbase +kylin.metadata.url= + +# The storage for final cube file in hbase +kylin.storage.url=hbase + +# Temp folder in hdfs, make sure user has the right access to the hdfs directory +kylin.hdfs.working.dir=/kylin + +kylin.job.mapreduce.default.reduce.input.mb=500 + +# If true, job engine will not assume that hadoop CLI reside on the same server as it self +# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password +kylin.job.run.as.remote.cmd=false + +# Only necessary when kylin.job.run.as.remote.cmd=true +kylin.job.remote.cli.hostname= + +# Only necessary when kylin.job.run.as.remote.cmd=true +kylin.job.remote.cli.username= + +# Only necessary when kylin.job.run.as.remote.cmd=true +kylin.job.remote.cli.password= + +# Used by test cases to prepare synthetic data for sample cube +kylin.job.remote.cli.working.dir=/tmp/kylin + +# Max count of concurrent jobs running +kylin.job.concurrent.max.limit=10 + +# Time interval to check hadoop job status +kylin.job.yarn.app.rest.check.interval.seconds=10 + +#default compression codec for htable,snappy,lzo,gzip,lz4 +kylin.hbase.default.compression.codec=gzip + +# test for hadoop job config override +kylin.job.mr.config.override.test1=test1 +kylin.job.mr.config.override.test2=test2 + + +kylin.security.profile=testing +## Config for Restful APP ## +# database connection settings: +ldap.server= +ldap.username= +ldap.password= +ldap.user.searchBase= +ldap.user.searchPattern= +ldap.user.groupSearchBase= +ldap.service.searchBase=OU= +ldap.service.searchPattern= +ldap.service.groupSearchBase= +acl.adminRole= +acl.defaultRole= +ganglia.group= +ganglia.port=8664 + + +
