Revert "KYLIN-2816 support hive unicode column name" This reverts commit 3712464e5e09c66b4c17cdb0c19e420c61c13358.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/487a854b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/487a854b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/487a854b Branch: refs/heads/master Commit: 487a854bfece7b9fc72b0118a40d51100eca8463 Parents: 81e9f57 Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Aug 28 19:19:24 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Tue Aug 29 10:36:10 2017 +0800 ---------------------------------------------------------------------- .../kylin/common/util/HiveCmdBuilder.java | 5 ++-- .../kylin/common/util/HiveCmdBuilderTest.java | 2 +- .../org/apache/kylin/job/JoinedFlatTable.java | 28 ++++---------------- .../apache/kylin/source/hive/HiveMRInput.java | 2 +- 4 files changed, 9 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index 191bbc0..2f6b9a0 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -66,12 +66,11 @@ public class HiveCmdBuilder { switch (clientMode) { case CLI: - // use single-quote to ignore the executing back-ticks surrounded column name in shell - buf.append("hive -e \'"); + buf.append("hive -e \""); for (String statement : statements) { buf.append(statement).append("\n"); } - buf.append("\'"); + buf.append("\""); buf.append(parseProps()); break; case BEELINE: http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java index d2938b5..d69d4d2 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java @@ -60,7 +60,7 @@ public class HiveCmdBuilderTest { hiveCmdBuilder.addStatement("SHOW\n TABLES;"); hiveCmdBuilder.setHiveConfProps(hiveProps); hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite); - assertEquals("hive -e \'USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\' --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build()); + assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build()); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 824c693..acb29e1 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -42,8 +42,6 @@ import org.apache.kylin.metadata.model.TblColRef; public class JoinedFlatTable { - public static final String BACKTICK = "`"; - public static String getTableDir(IJoinedFlatTableDesc flatDesc, String storageDfsDir) { return storageDfsDir + "/" + flatDesc.getTableName(); } @@ -77,7 +75,7 @@ public class JoinedFlatTable { ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); } ddl.append("STORED AS " + format + "\n"); - ddl.append("LOCATION '\"'\"'" + getTableDir(flatDesc, storageDfsDir) + "'\"'\"';").append("\n"); + ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + "';").append("\n"); return ddl.toString(); } @@ -130,15 +128,10 @@ public class JoinedFlatTable { sql.append(","); } String colTotalName = String.format("%s.%s", col.getTableRef().getTableName(), col.getName()); - String expressionInSourceDB = col.getExpressionInSourceDB(); - if (expressionInSourceDB.contains(".")) { - // surround column name with back-tick, to support unicode column name - expressionInSourceDB = expressionInSourceDB.replace(".", "." + BACKTICK) + BACKTICK; - } if (skipAsList.contains(colTotalName)) { - sql.append(expressionInSourceDB + sep); + sql.append(col.getExpressionInSourceDB() + sep); } else { - sql.append(expressionInSourceDB + " as " + colName(col) + sep); + sql.append(col.getExpressionInSourceDB() + " as " + colName(col) + sep); } } appendJoinStatement(flatDesc, sql, singleLine); @@ -146,13 +139,7 @@ public class JoinedFlatTable { return sql.toString(); } - /** - * @deprecated - * @param flatDesc - * @param outputDir - * @return - */ - static String generateCountDataStatement(IJoinedFlatTableDesc flatDesc, final String outputDir) { + public static String generateCountDataStatement(IJoinedFlatTableDesc flatDesc, final String outputDir) { final StringBuilder sql = new StringBuilder(); final TableRef rootTbl = flatDesc.getDataModel().getRootFactTable(); sql.append("dfs -mkdir -p " + outputDir + ";\n"); @@ -246,13 +233,8 @@ public class JoinedFlatTable { } } - /** - * Column name with `BACKTICK` - * @param col - * @return - */ private static String colName(TblColRef col) { - return BACKTICK + col.getTableAlias() + "_" + col.getName() + BACKTICK; + return col.getTableAlias() + "_" + col.getName(); } private static String getHiveDataType(String javaDataType) { http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 7dcf069..096134c 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -221,7 +221,7 @@ public class HiveMRInput implements IMRInput { createIntermediateTableHql.append("DROP TABLE IF EXISTS " + intermediate + ";\n"); createIntermediateTableHql .append("CREATE EXTERNAL TABLE IF NOT EXISTS " + intermediate + " LIKE " + identity + "\n"); - createIntermediateTableHql.append("LOCATION '\"'\"'" + jobWorkingDir + "/" + intermediate + "'\"'\"';\n"); + createIntermediateTableHql.append("LOCATION '" + jobWorkingDir + "/" + intermediate + "';\n"); createIntermediateTableHql .append("INSERT OVERWRITE TABLE " + intermediate + " SELECT * FROM " + identity + ";\n"); hiveCmdBuilder.addStatement(createIntermediateTableHql.toString());
