KYLIN-2577, refine hive configurations during cubing steps
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c152c51 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c152c51 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c152c51 Branch: refs/heads/master Commit: 7c152c515ce6cebe380c4e1b7c16d109192308db Parents: 0061900 Author: Cheng Wang <cheng.w...@kyligence.io> Authored: Thu May 18 11:05:34 2017 +0800 Committer: liyang-gmt8 <liy...@apache.org> Committed: Thu May 18 15:27:36 2017 +0800 ---------------------------------------------------------------------- build/bin/load-hive-conf.sh | 2 +- .../kylin/common/util/HiveCmdBuilder.java | 56 +++++++++++++++++++- .../kylin/common/util/HiveCmdBuilderTest.java | 2 +- .../kylin/job/engine/JobEngineConfig.java | 43 --------------- .../source/hive/CreateFlatHiveTableStep.java | 3 -- .../apache/kylin/source/hive/HiveMRInput.java | 4 -- 6 files changed, 57 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/build/bin/load-hive-conf.sh ---------------------------------------------------------------------- diff --git a/build/bin/load-hive-conf.sh b/build/bin/load-hive-conf.sh index a5046f0..a42cd0c 100644 --- a/build/bin/load-hive-conf.sh +++ b/build/bin/load-hive-conf.sh @@ -11,5 +11,5 @@ hive_conf_prop="${KYLIN_HOME}/logs/hive_props" rm -rf ${hive_conf_prop} export ENABLE_CHECK_ENV=false ${dir}/kylin.sh org.apache.kylin.tool.HiveConfigCLI ${hive_conf_dir} ${hive_conf_prop} -[[ 0 == $? ]] || quit "Error, can not parse ${hive_conf_dir} and can not apply it to hive relevant check." +[[ 0 == $? ]] || quit "Can not parse xml file: ${hive_conf_dir}, please check it." hive_conf_properties=`cat ${hive_conf_prop}` \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index 255867a..2f6b9a0 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -27,17 +27,25 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; import com.google.common.collect.Lists; public class HiveCmdBuilder { private static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class); + public static final String HIVE_CONF_FILENAME = "kylin_hive_conf"; + public enum HiveClientMode { CLI, BEELINE } @@ -50,6 +58,7 @@ public class HiveCmdBuilder { public HiveCmdBuilder() { kylinConfig = KylinConfig.getInstanceFromEnv(); clientMode = HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase()); + loadHiveConfiguration(); } public String build() { @@ -75,8 +84,8 @@ public class HiveCmdBuilder { bw.newLine(); } buf.append("beeline "); - buf.append(parseProps()); buf.append(kylinConfig.getHiveBeelineParams()); + buf.append(parseProps()); buf.append(" -f "); buf.append(tmpHql.getAbsolutePath()); buf.append(";ret_code=$?;rm -f "); @@ -123,6 +132,7 @@ public class HiveCmdBuilder { } public void setHiveConfProps(Map<String, String> hiveConfProps) { + this.hiveConfProps.clear(); this.hiveConfProps.putAll(hiveConfProps); } @@ -144,4 +154,48 @@ public class HiveCmdBuilder { public String toString() { return build(); } + + private void loadHiveConfiguration() { + + File hiveConfFile; + String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml"); + String path = System.getProperty(KylinConfig.KYLIN_CONF); + + if (StringUtils.isNotEmpty(path)) { + hiveConfFile = new File(path, hiveConfFileName); + } else { + path = KylinConfig.getKylinHome(); + if (StringUtils.isEmpty(path)) { + logger.error("KYLIN_HOME is not set, can not locate hive conf: {}.xml", HIVE_CONF_FILENAME); + return; + } + hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName); + } + + if (hiveConfFile == null || !hiveConfFile.exists()) { + throw new RuntimeException("Failed to read " + HIVE_CONF_FILENAME + ".xml"); + } + + String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath()); + + try { + File file = new File(fileUrl); + if (file.exists()) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(file); + NodeList nl = doc.getElementsByTagName("property"); + hiveConfProps.clear(); + for (int i = 0; i < nl.getLength(); i++) { + String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue(); + String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue(); + if (!key.equals("tmpjars")) { + hiveConfProps.put(key, value); + } + } + } + } catch (Exception e) { + throw new RuntimeException("Failed to parse hive conf file ", e); + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java index d847575..d69d4d2 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java @@ -75,7 +75,7 @@ public class HiveCmdBuilderTest { hiveCmdBuilder.addStatement("SHOW\n TABLES;"); String cmd = hiveCmdBuilder.build(); - assertTrue(cmd.startsWith("beeline -u jdbc_url -f")); + assertTrue(cmd.startsWith("beeline -u jdbc_url")); String hqlFile = cmd.substring(cmd.lastIndexOf("-f ") + 3).trim(); hqlFile = hqlFile.substring(0, hqlFile.length() - ";exit $ret_code".length()); http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java index 260f817..c9ac583 100644 --- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java +++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java @@ -20,19 +20,12 @@ package org.apache.kylin.job.engine; import java.io.File; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.OptionsHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; /** * @author ysong1 @@ -40,7 +33,6 @@ import org.w3c.dom.NodeList; public class JobEngineConfig { private static final Logger logger = LoggerFactory.getLogger(JobEngineConfig.class); public static final String HADOOP_JOB_CONF_FILENAME = "kylin_job_conf"; - public static final String HIVE_CONF_FILENAME = "kylin_hive_conf"; public static final String DEFAUL_JOB_CONF_SUFFIX = ""; public static final String IN_MEM_JOB_CONF_SUFFIX = "inmem"; @@ -102,41 +94,6 @@ public class JobEngineConfig { return path; } - public String getHiveConfFilePath() { - String hiveConfFile = (HIVE_CONF_FILENAME + ".xml"); - - File jobConfig = getJobConfig(hiveConfFile); - if (jobConfig == null || !jobConfig.exists()) { - - logger.error("fail to locate " + HIVE_CONF_FILENAME + ".xml"); - throw new RuntimeException("fail to locate " + HIVE_CONF_FILENAME + ".xml"); - } - return OptionsHelper.convertToFileURL(jobConfig.getAbsolutePath()); - } - - public Map<String, String> getHivePropsFromFile() { - Map<String, String> props = new HashMap<>(); - try { - File file = new File(getHiveConfFilePath()); - if (file.exists()) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document doc = builder.parse(file); - NodeList nl = doc.getElementsByTagName("property"); - for (int i = 0; i < nl.getLength(); i++) { - String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue(); - String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue(); - if (!key.equals("tmpjars")) { - props.put(key, value); - } - } - } - } catch (Exception e) { - throw new RuntimeException("Failed to parse hive conf file ", e); - } - return props; - } - // there should be no setters private final KylinConfig config; http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java index 9e13df8..f28f3c7 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java @@ -26,7 +26,6 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; import org.apache.kylin.job.common.PatternedLogger; -import org.apache.kylin.job.engine.JobEngineConfig; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; @@ -43,8 +42,6 @@ public class CreateFlatHiveTableStep extends AbstractExecutable { protected void createFlatHiveTable(KylinConfig config) throws IOException { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); - final JobEngineConfig jobEngineConfig = new JobEngineConfig(config); - hiveCmdBuilder.setHiveConfProps(jobEngineConfig.getHivePropsFromFile()); hiveCmdBuilder.overwriteHiveProps(config.getHiveConfigOverride()); hiveCmdBuilder.addStatement(getInitStatement()); hiveCmdBuilder.addStatement(getCreateTableStatement()); http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 6c542ab..e388074 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -257,7 +257,6 @@ public class HiveMRInput implements IMRInput { private void redistributeTable(KylinConfig config, int numReducers) throws IOException { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); - hiveCmdBuilder.setHiveConfProps(new JobEngineConfig(config).getHivePropsFromFile()); hiveCmdBuilder.addStatement(getInitStatement()); hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n"); hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n"); @@ -375,13 +374,10 @@ public class HiveMRInput implements IMRInput { final String hiveTable = this.getIntermediateTableIdentity(); if (config.isHiveKeepFlatTable() == false && StringUtils.isNotEmpty(hiveTable)) { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); - hiveCmdBuilder.setHiveConfProps(new JobEngineConfig(config).getHivePropsFromFile()); hiveCmdBuilder.addStatement("USE " + config.getHiveDatabaseForIntermediateTable() + ";"); hiveCmdBuilder.addStatement("DROP TABLE IF EXISTS " + hiveTable + ";"); - config.getCliCommandExecutor().execute(hiveCmdBuilder.build()); output.append("Hive table " + hiveTable + " is dropped. \n"); - rmdirOnHDFS(getExternalDataPath()); output.append("Hive table " + hiveTable + " external data path " + getExternalDataPath() + " is deleted. \n"); }
