KYLIN-2816 wrap columns with backtick
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74c904e4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74c904e4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74c904e4 Branch: refs/heads/master Commit: 74c904e4eedc224b07c9b6463ab0b63faddd4a21 Parents: b778f33 Author: tttMelody <245915...@qq.com> Authored: Mon Aug 28 14:19:36 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Mon Aug 28 15:25:27 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/job/JoinedFlatTable.java | 77 ++++++++++++++++++-- .../kylin/job/TestWrapColumnWithBacktick.java | 57 +++++++++++++++ .../metadata/model/tool/CalciteParser.java | 8 +- 3 files changed, 135 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/74c904e4/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 824c693..4ac5a3a 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -21,11 +21,17 @@ package org.apache.kylin.job; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.job.engine.JobEngineConfig; import org.apache.kylin.metadata.model.DataModelDesc; @@ -35,6 +41,9 @@ import org.apache.kylin.metadata.model.JoinTableDesc; import org.apache.kylin.metadata.model.PartitionDesc; import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.model.tool.CalciteParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @@ -43,6 +52,7 @@ import org.apache.kylin.metadata.model.TblColRef; public class JoinedFlatTable { public static final String BACKTICK = "`"; + private static final Logger logger = LoggerFactory.getLogger(JoinedFlatTable.class); public static String getTableDir(IJoinedFlatTableDesc flatDesc, String storageDfsDir) { return storageDfsDir + "/" + flatDesc.getTableName(); @@ -130,11 +140,7 @@ public class JoinedFlatTable { sql.append(","); } String colTotalName = String.format("%s.%s", col.getTableRef().getTableName(), col.getName()); - String expressionInSourceDB = col.getExpressionInSourceDB(); - if (expressionInSourceDB.contains(".")) { - // surround column name with back-tick, to support unicode column name - expressionInSourceDB = expressionInSourceDB.replace(".", "." + BACKTICK) + BACKTICK; - } + String expressionInSourceDB = quoteOriginalColumnWithBacktick(col.getExpressionInSourceDB()); if (skipAsList.contains(colTotalName)) { sql.append(expressionInSourceDB + sep); } else { @@ -277,4 +283,65 @@ public class JoinedFlatTable { return sql.toString(); } + /** + * quote column name with back-tick, to support unicode column name + * + * @param sourceSQL + * @return + */ + static String quoteOriginalColumnWithBacktick(String sourceSQL) { + StringBuilder result = new StringBuilder(sourceSQL); + try { + List<Pair<Integer, Integer>> replacePoses = new ArrayList<>(); + for (SqlIdentifier id : SqlIdentifierVisitor.getAllSqlIdentifier(CalciteParser.getExpNode(sourceSQL))) { + replacePoses.add(CalciteParser.getReplacePos(id.getComponentParserPosition(1), "select " + sourceSQL + " from t")); + } + + // latter replace position in the front of the list. + Collections.sort(replacePoses, new Comparator<Pair<Integer, Integer>>() { + @Override + public int compare(Pair<Integer, Integer> o1, Pair<Integer, Integer> o2) { + return -(o1.getSecond() - o2.getSecond()); + } + }); + + // cuz the pos is add "select "'s size, so minus 7 offset. + final int OFFSET = 7; + for (Pair<Integer, Integer> replacePos : replacePoses) { + result.insert(replacePos.getSecond() - OFFSET, "`"); + result.insert(replacePos.getFirst() - OFFSET, "`"); + } + } catch (Exception e) { + logger.error("quote original column with backtick fail.Return origin SQL." + e.getMessage()); + } + + return result.toString(); + } + + + static class SqlIdentifierVisitor extends SqlBasicVisitor<SqlNode> { + private List<SqlIdentifier> sqlIdentifier; + + SqlIdentifierVisitor() { + this.sqlIdentifier = new ArrayList<>(); + } + + List<SqlIdentifier> getSqlIdentifier() { + return sqlIdentifier; + } + + static List<SqlIdentifier> getAllSqlIdentifier(SqlNode node) { + SqlIdentifierVisitor siv = new SqlIdentifierVisitor(); + node.accept(siv); + return siv.getSqlIdentifier(); + } + + @Override + public SqlNode visit(SqlIdentifier id) { + if (id.names.size() == 2) { + sqlIdentifier.add(id); + } + return null; + } + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/74c904e4/core-job/src/test/java/org/apache/kylin/job/TestWrapColumnWithBacktick.java ---------------------------------------------------------------------- diff --git a/core-job/src/test/java/org/apache/kylin/job/TestWrapColumnWithBacktick.java b/core-job/src/test/java/org/apache/kylin/job/TestWrapColumnWithBacktick.java new file mode 100644 index 0000000..0c55ba7 --- /dev/null +++ b/core-job/src/test/java/org/apache/kylin/job/TestWrapColumnWithBacktick.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.job; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestWrapColumnWithBacktick { + @Test + public void testWrapColumnNameInFlatTable() { + String input1 = "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"; + String expected1 = "SELLER_ACCOUNT.`ACCOUNT_SELLER_LEVEL`"; + String input2 = "year(TEST_KYLIN_FACT.CAL_DT)"; + String expected2 = "year(TEST_KYLIN_FACT.`CAL_DT`)"; + String input3 = "SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,0,1)"; + String expected3 = "SUBSTR(SELLER_ACCOUNT.`ACCOUNT_COUNTRY`,0,1)"; + String input4 = "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME)"; + String expected4 = "CONCAT(SELLER_ACCOUNT.`ACCOUNT_ID`, SELLER_COUNTRY.`NAME`)"; + String input5 = "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL*1.1"; + String expected5 = "SELLER_ACCOUNT.`ACCOUNT_SELLER_LEVEL`*1.1"; + String input6 = "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL*SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"; + String expected6 = "SELLER_ACCOUNT.`ACCOUNT_SELLER_LEVEL`*SELLER_ACCOUNT.`ACCOUNT_SELLER_LEVEL`"; + String input7 = "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, 'HELLO.WORLD')"; + String expected7 = "CONCAT(SELLER_ACCOUNT.`ACCOUNT_ID`, 'HELLO.WORLD')"; + String input8 = "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_ACCOUNT.ACCOUNT_ID2)"; + String expected8 = "CONCAT(SELLER_ACCOUNT.`ACCOUNT_ID`, SELLER_ACCOUNT.`ACCOUNT_ID2`)"; + String input9 = "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_ACCOUNT.ACCOUNT_ID2, SELLER_ACCOUNT.ACCOUNT_ID3)"; + String expected9 = "CONCAT(SELLER_ACCOUNT.`ACCOUNT_ID`, SELLER_ACCOUNT.`ACCOUNT_ID2`, SELLER_ACCOUNT.`ACCOUNT_ID3`)"; + + assertEquals(expected1, JoinedFlatTable.quoteOriginalColumnWithBacktick(input1)); + assertEquals(expected2, JoinedFlatTable.quoteOriginalColumnWithBacktick(input2)); + assertEquals(expected3, JoinedFlatTable.quoteOriginalColumnWithBacktick(input3)); + assertEquals(expected4, JoinedFlatTable.quoteOriginalColumnWithBacktick(input4)); + assertEquals(expected5, JoinedFlatTable.quoteOriginalColumnWithBacktick(input5)); + assertEquals(expected6, JoinedFlatTable.quoteOriginalColumnWithBacktick(input6)); + assertEquals(expected7, JoinedFlatTable.quoteOriginalColumnWithBacktick(input7)); + assertEquals(expected8, JoinedFlatTable.quoteOriginalColumnWithBacktick(input8)); + assertEquals(expected9, JoinedFlatTable.quoteOriginalColumnWithBacktick(input9)); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/74c904e4/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java index fdad33a..09f2f28 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java @@ -32,11 +32,11 @@ import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.calcite.sql.util.SqlVisitor; +import org.apache.kylin.common.util.Pair; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.kylin.common.util.Pair; public class CalciteParser { public static SqlNode parse(String sql) throws SqlParseException { @@ -127,11 +127,15 @@ public class CalciteParser { } public static Pair<Integer, Integer> getReplacePos(SqlNode node, String inputSql) { + SqlParserPos pos = node.getParserPosition(); + return getReplacePos(pos, inputSql); + } + + public static Pair<Integer, Integer> getReplacePos(SqlParserPos pos, String inputSql) { if (inputSql == null) { return Pair.newPair(0, 0); } String[] lines = inputSql.split("\n"); - SqlParserPos pos = node.getParserPosition(); int lineStart = pos.getLineNum(); int lineEnd = pos.getEndLineNum(); int columnStart = pos.getColumnNum() - 1;
