KYLIN-2816 support hive unicode column name
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28a07ebd Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28a07ebd Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28a07ebd Branch: refs/heads/master Commit: 28a07ebd3983c325c64c0642e7dedbf32c284f39 Parents: e2e0102 Author: Billy(Yiming) Liu <liuyiming....@gmail.com> Authored: Sun Aug 27 18:55:03 2017 +0800 Committer: GitHub <nore...@github.com> Committed: Sun Aug 27 18:55:03 2017 +0800 ---------------------------------------------------------------------- .../kylin/common/util/HiveCmdBuilder.java | 5 ++-- .../kylin/common/util/HiveCmdBuilderTest.java | 2 +- .../org/apache/kylin/job/JoinedFlatTable.java | 28 ++++++++++++++++---- .../apache/kylin/source/hive/HiveMRInput.java | 2 +- 4 files changed, 28 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index 2f6b9a0..191bbc0 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -66,11 +66,12 @@ public class HiveCmdBuilder { switch (clientMode) { case CLI: - buf.append("hive -e \""); + // use single-quote to ignore the executing back-ticks surrounded column name in shell + buf.append("hive -e \'"); for (String statement : statements) { buf.append(statement).append("\n"); } - buf.append("\""); + buf.append("\'"); buf.append(parseProps()); break; case BEELINE: http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java index d69d4d2..d2938b5 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java @@ -60,7 +60,7 @@ public class HiveCmdBuilderTest { hiveCmdBuilder.addStatement("SHOW\n TABLES;"); hiveCmdBuilder.setHiveConfProps(hiveProps); hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite); - assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build()); + assertEquals("hive -e \'USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\' --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build()); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index acb29e1..824c693 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -42,6 +42,8 @@ import org.apache.kylin.metadata.model.TblColRef; public class JoinedFlatTable { + public static final String BACKTICK = "`"; + public static String getTableDir(IJoinedFlatTableDesc flatDesc, String storageDfsDir) { return storageDfsDir + "/" + flatDesc.getTableName(); } @@ -75,7 +77,7 @@ public class JoinedFlatTable { ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); } ddl.append("STORED AS " + format + "\n"); - ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + "';").append("\n"); + ddl.append("LOCATION '\"'\"'" + getTableDir(flatDesc, storageDfsDir) + "'\"'\"';").append("\n"); return ddl.toString(); } @@ -128,10 +130,15 @@ public class JoinedFlatTable { sql.append(","); } String colTotalName = String.format("%s.%s", col.getTableRef().getTableName(), col.getName()); + String expressionInSourceDB = col.getExpressionInSourceDB(); + if (expressionInSourceDB.contains(".")) { + // surround column name with back-tick, to support unicode column name + expressionInSourceDB = expressionInSourceDB.replace(".", "." + BACKTICK) + BACKTICK; + } if (skipAsList.contains(colTotalName)) { - sql.append(col.getExpressionInSourceDB() + sep); + sql.append(expressionInSourceDB + sep); } else { - sql.append(col.getExpressionInSourceDB() + " as " + colName(col) + sep); + sql.append(expressionInSourceDB + " as " + colName(col) + sep); } } appendJoinStatement(flatDesc, sql, singleLine); @@ -139,7 +146,13 @@ public class JoinedFlatTable { return sql.toString(); } - public static String generateCountDataStatement(IJoinedFlatTableDesc flatDesc, final String outputDir) { + /** + * @deprecated + * @param flatDesc + * @param outputDir + * @return + */ + static String generateCountDataStatement(IJoinedFlatTableDesc flatDesc, final String outputDir) { final StringBuilder sql = new StringBuilder(); final TableRef rootTbl = flatDesc.getDataModel().getRootFactTable(); sql.append("dfs -mkdir -p " + outputDir + ";\n"); @@ -233,8 +246,13 @@ public class JoinedFlatTable { } } + /** + * Column name with `BACKTICK` + * @param col + * @return + */ private static String colName(TblColRef col) { - return col.getTableAlias() + "_" + col.getName(); + return BACKTICK + col.getTableAlias() + "_" + col.getName() + BACKTICK; } private static String getHiveDataType(String javaDataType) { http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 096134c..7dcf069 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -221,7 +221,7 @@ public class HiveMRInput implements IMRInput { createIntermediateTableHql.append("DROP TABLE IF EXISTS " + intermediate + ";\n"); createIntermediateTableHql .append("CREATE EXTERNAL TABLE IF NOT EXISTS " + intermediate + " LIKE " + identity + "\n"); - createIntermediateTableHql.append("LOCATION '" + jobWorkingDir + "/" + intermediate + "';\n"); + createIntermediateTableHql.append("LOCATION '\"'\"'" + jobWorkingDir + "/" + intermediate + "'\"'\"';\n"); createIntermediateTableHql .append("INSERT OVERWRITE TABLE " + intermediate + " SELECT * FROM " + identity + ";\n"); hiveCmdBuilder.addStatement(createIntermediateTableHql.toString());
