Repository: kylin Updated Branches: refs/heads/master-hadoop3.0 [created] dcb7cf47c
minor, move QueryTransformer to Query module Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/037dd474 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/037dd474 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/037dd474 Branch: refs/heads/master-hadoop3.0 Commit: 037dd474980b2e33ee80674d4a5de2434a033e85 Parents: e9ec116 Author: lidongsjtu <[email protected]> Authored: Tue Apr 11 18:47:02 2017 +0800 Committer: Dong Li <[email protected]> Committed: Tue Apr 11 19:00:34 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/ClassUtil.java | 2 + .../org/apache/kylin/cube/model/CubeDesc.java | 15 +- .../test_case_data/localmeta/kylin.properties | 2 +- .../query/util/CognosParenthesesEscape.java | 82 ++++++++ .../query/util/KeywordDefaultDirtyHack.java | 33 ++++ .../org/apache/kylin/query/util/QueryUtil.java | 190 +++++++++++++++++++ .../query/util/CognosParentesesEscapeTest.java | 61 ++++++ .../apache/kylin/query/util/QueryUtilTest.java | 61 ++++++ .../src/test/resources/query/cognos/query01.sql | 28 +++ .../resources/query/cognos/query01.sql.expected | 28 +++ .../apache/kylin/rest/service/QueryService.java | 4 +- .../rest/util/CognosParenthesesEscape.java | 83 -------- .../rest/util/KeywordDefaultDirtyHack.java | 35 ---- .../org/apache/kylin/rest/util/QueryUtil.java | 190 ------------------- .../rest/util/CognosParentesesEscapeTest.java | 61 ------ .../apache/kylin/rest/util/QueryUtilTest.java | 65 ------- .../src/test/resources/query/cognos/query01.sql | 28 --- .../resources/query/cognos/query01.sql.expected | 28 --- .../rest/controller/QueryControllerTest.java | 2 +- 19 files changed, 502 insertions(+), 496 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java index 35a3277..0eb1af5 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java @@ -65,6 +65,8 @@ public class ClassUtil { classRenameMap.put("org.apache.kylin.job.cube.MergeDictionaryStep", "org.apache.kylin.engine.mr.steps.MergeDictionaryStep"); classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterBuildStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep"); classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterMergeStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterMergeStep"); + classRenameMap.put("org.apache.kylin.rest.util.KeywordDefaultDirtyHack", "org.apache.kylin.query.util.KeywordDefaultDirtyHack"); + classRenameMap.put("org.apache.kylin.rest.util.CognosParenthesesEscape", "org.apache.kylin.query.util.CognosParenthesesEscape"); } @SuppressWarnings("unchecked") http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index b391055..820a172 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -97,7 +97,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } } - public enum DeriveType implements java.io.Serializable{ + public enum DeriveType implements java.io.Serializable { LOOKUP, PK_FK, EXTENDED_COLUMN } @@ -485,7 +485,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { logger.info("checkSignature on {} is skipped as the its version {} is different from kylin version {}", getName(), cubeVersion, kylinVersion); return true; } - + if (kylinVersion.isCompatibleWith(cubeVersion) && !kylinVersion.isSignatureCompatibleWith(cubeVersion)) { logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), cubeVersion); return true; @@ -532,6 +532,17 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } } + public void deInit() { + config = null; + model = null; + allColumns = new LinkedHashSet<>(); + allColumnDescs = new LinkedHashSet<>(); + dimensionColumns = new LinkedHashSet<>(); + derivedToHostMap = Maps.newHashMap(); + hostToDerivedMap = Maps.newHashMap(); + extendedColumnToHosts = Maps.newHashMap(); + } + public void init(KylinConfig config) { this.errors.clear(); http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/examples/test_case_data/localmeta/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index 9f7b24c..6fef7f7 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -80,7 +80,7 @@ kylin.engine.mr.uhc-reducer-count=3 ### QUERY ### -kylin.query.transformers=org.apache.kylin.rest.util.KeywordDefaultDirtyHack +kylin.query.transformers=org.apache.kylin.query.util.KeywordDefaultDirtyHack ### SECURITY ### http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java b/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java new file mode 100644 index 0000000..70ab8da --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query.util; + +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.kylin.common.util.Pair; + +/** + * from (a join b on a.x = b.y) join c + * + * similar in https://issues.apache.org/jira/browse/CALCITE-35 + * + * we'll find such pattern and remove the parentheses + */ +public class CognosParenthesesEscape implements QueryUtil.IQueryTransformer { + + private static final String S0 = "\\s*"; + private static final String S1 = "\\s"; + private static final String SM = "\\s+"; + private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+"; + private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS = TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?"; + private static final String JOIN = "(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server + private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME + S0 + "=" + S0 + TABLE_OR_COLUMN_NAME; + private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // ( + TABLE_NAME_WITH_OPTIONAL_ALIAS + // a + JOIN + SM + // join + TABLE_NAME_WITH_OPTIONAL_ALIAS + //b + SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*" + // on a.x = b.y [and a.x2 = b.y2] + S0 + "\\)";// ) + private static final Pattern PARENTTHESES_PATTERN = Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE); + + private static int identifierNum = 0; + + @Override + public String transform(String sql) { + Matcher m; + List<Pair<String, String>> matches = new LinkedList<>(); + while (true) { + m = PARENTTHESES_PATTERN.matcher(sql); + if (!m.find()) + break; + + String oneParentheses = m.group(0); + String identifier = generateRandomName(); + matches.add(new Pair<String, String>(identifier, oneParentheses.substring(1, oneParentheses.length() - 1))); + sql = sql.substring(0, m.start()) + identifier + sql.substring(m.end()); + } + + for (int i = matches.size() - 1; i >= 0; i--) { + sql = sql.replaceAll(matches.get(i).getKey(), matches.get(i).getValue()); + } + + return sql; + } + + private String generateRandomName() { + UUID uuid = UUID.randomUUID(); + return uuid.toString().replace("-", "_") + "_" + (identifierNum++); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java b/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java new file mode 100644 index 0000000..e1398f6 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query.util; + +public class KeywordDefaultDirtyHack implements QueryUtil.IQueryTransformer { + + @Override + public String transform(String sql) { + // KYLIN-2108, DEFAULT is hive default database, but a sql keyword too, needs quote + sql = sql.replace("DEFAULT.", "\"DEFAULT\"."); + sql = sql.replace("default.", "\"default\"."); + sql = sql.replace("defaultCatalog.", ""); + + return sql; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java new file mode 100644 index 0000000..d48a26f --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query.util; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.ClassUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +/** + */ +public class QueryUtil { + + protected static final Logger logger = LoggerFactory.getLogger(QueryUtil.class); + + private static List<IQueryTransformer> queryTransformers; + + public interface IQueryTransformer { + String transform(String sql); + } + + public static String massageSql(String sql) { + return massageSql(sql, 0, 0); + } + + public static String massageSql(String sql, int limit, int offset) { + sql = sql.trim(); + sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator")); + + while (sql.endsWith(";")) + sql = sql.substring(0, sql.length() - 1); + + if (limit > 0 && !sql.toLowerCase().contains("limit")) { + sql += ("\nLIMIT " + limit); + } + + if (offset > 0 && !sql.toLowerCase().contains("offset")) { + sql += ("\nOFFSET " + offset); + } + + // customizable SQL transformation + if (queryTransformers == null) { + initQueryTransformers(); + } + for (IQueryTransformer t : queryTransformers) { + sql = t.transform(sql); + } + return sql; + } + + private static void initQueryTransformers() { + List<IQueryTransformer> transformers = Lists.newArrayList(); + transformers.add(new DefaultQueryTransformer()); + + String[] classes = KylinConfig.getInstanceFromEnv().getQueryTransformers(); + for (String clz : classes) { + try { + IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz); + transformers.add(t); + } catch (Exception e) { + logger.error("Failed to init query transformer", e); + } + } + queryTransformers = transformers; + } + + // correct sick / invalid SQL + private static class DefaultQueryTransformer implements IQueryTransformer { + + private static final String S0 = "\\s*"; + private static final String S1 = "\\s"; + private static final String SM = "\\s+"; + private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + "GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + "1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!=" + S0, Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_INTERVAL = Pattern.compile("interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + "(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE); + private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE); + + @Override + public String transform(String sql) { + Matcher m; + + // Case fn{ EXTRACT(...) } + // Use non-greedy regrex matching to remove escape functions + while (true) { + m = PTN_HAVING_ESCAPE_FUNCTION.matcher(sql); + if (!m.find()) + break; + sql = sql.substring(0, m.start()) + m.group(1) + sql.substring(m.end()); + } + + // Case: HAVING COUNT(1)>0 without Group By + // Tableau generates: SELECT SUM(1) AS "COL" FROM "VAC_SW" HAVING + // COUNT(1)>0 + m = PTN_HAVING_COUNT_GREATER_THAN_ZERO.matcher(sql); + if (m.find() && PTN_GROUP_BY.matcher(sql).find() == false) { + sql = sql.substring(0, m.start()) + " " + sql.substring(m.end()); + } + + // Case: SUM(1) + // Replace it with COUNT(1) + while (true) { + m = PTN_SUM_1.matcher(sql); + if (!m.find()) + break; + sql = sql.substring(0, m.start()) + " COUNT(1) " + sql.substring(m.end()); + } + + // Case: != + // Replace it with <> + while (true) { + m = PTN_NOT_EQ.matcher(sql); + if (!m.find()) + break; + sql = sql.substring(0, m.start()) + " <> " + sql.substring(m.end()); + } + + // ( date '2001-09-28' + interval floor(1) day ) generated by cognos + // calcite only recognizes date '2001-09-28' + interval '1' day + while (true) { + m = PTN_INTERVAL.matcher(sql); + if (!m.find()) + break; + + int value = (int) Math.floor(Double.valueOf(m.group(2))); + sql = sql.substring(0, m.start(1)) + "'" + value + "'" + sql.substring(m.end(3)); + } + + return sql; + } + + } + + public static String makeErrorMsgUserFriendly(Throwable e) { + String msg = e.getMessage(); + + // pick ParseException error message if possible + Throwable cause = e; + while (cause != null) { + if (cause.getClass().getName().contains("ParseException")) { + msg = cause.getMessage(); + break; + } + cause = cause.getCause(); + } + + return makeErrorMsgUserFriendly(msg); + } + + public static String makeErrorMsgUserFriendly(String errorMsg) { + try { + // make one line + errorMsg = errorMsg.replaceAll("\\s", " "); + + // move cause to be ahead of sql, calcite creates the message pattern below + Pattern pattern = Pattern.compile("error while executing SQL \"(.*)\":(.*)"); + Matcher matcher = pattern.matcher(errorMsg); + if (matcher.find()) { + return matcher.group(2).trim() + "\n" + "while executing SQL: \"" + matcher.group(1).trim() + "\""; + } else + return errorMsg; + } catch (Exception e) { + return errorMsg; + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/test/java/org/apache/kylin/query/util/CognosParentesesEscapeTest.java ---------------------------------------------------------------------- diff --git a/query/src/test/java/org/apache/kylin/query/util/CognosParentesesEscapeTest.java b/query/src/test/java/org/apache/kylin/query/util/CognosParentesesEscapeTest.java new file mode 100644 index 0000000..1f62d71 --- /dev/null +++ b/query/src/test/java/org/apache/kylin/query/util/CognosParentesesEscapeTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.query.util; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Collection; + +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Test; + +public class CognosParentesesEscapeTest { + + @Test + public void basicTest() { + CognosParenthesesEscape escape = new CognosParenthesesEscape(); + String data = "((a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3) inner join c as cc on a.x1=cc.z1 ) join d dd on a.x1=d.w1 and a.x2 =d.w2 "; + String expected = "a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3 inner join c as cc on a.x1=cc.z1 join d dd on a.x1=d.w1 and a.x2 =d.w2 "; + String transformed = escape.transform(data); + Assert.assertEquals(expected, transformed); + } + + @Test + public void advancedTest() throws IOException { + CognosParenthesesEscape escape = new CognosParenthesesEscape(); + String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql"), Charset.defaultCharset()); + String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql.expected"), Charset.defaultCharset()); + String transformed = escape.transform(query); + //System.out.println(transformed); + Assert.assertEquals(expected, transformed); + } + + @Test + public void proguardTest() throws IOException { + CognosParenthesesEscape escape = new CognosParenthesesEscape(); + Collection<File> files = FileUtils.listFiles(new File("../kylin-it/src/test/resources"), new String[] { "sql" }, true); + for (File f : files) { + System.out.println("checking " + f.getAbsolutePath()); + String query = FileUtils.readFileToString(f, Charset.defaultCharset()); + String transformed = escape.transform(query); + Assert.assertEquals(query, transformed); + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java ---------------------------------------------------------------------- diff --git a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java new file mode 100644 index 0000000..a1edd89 --- /dev/null +++ b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.query.util; + +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class QueryUtilTest extends LocalFileMetadataTestCase { + + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + @Test + public void testMassageSql() { + { + String sql = "select ( date '2001-09-28' + interval floor(1.2) day) from test_kylin_fact"; + String s = QueryUtil.massageSql(sql, 0, 0); + Assert.assertEquals("select ( date '2001-09-28' + interval '1' day) from test_kylin_fact", s); + } + { + String sql = "select ( date '2001-09-28' + interval floor(2) month) from test_kylin_fact group by ( date '2001-09-28' + interval floor(2) month)"; + String s = QueryUtil.massageSql(sql, 0, 0); + Assert.assertEquals("select ( date '2001-09-28' + interval '2' month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' month)", s); + } + } + + @Test + public void testKeywordDefaultDirtyHack() { + { + String sql = "select * from DEFAULT.TEST_KYLIN_FACT"; + String s = QueryUtil.massageSql(sql, 0, 0); + Assert.assertEquals("select * from \"DEFAULT\".TEST_KYLIN_FACT", s); + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/test/resources/query/cognos/query01.sql ---------------------------------------------------------------------- diff --git a/query/src/test/resources/query/cognos/query01.sql b/query/src/test/resources/query/cognos/query01.sql new file mode 100644 index 0000000..a76b4e1 --- /dev/null +++ b/query/src/test/resources/query/cognos/query01.sql @@ -0,0 +1,28 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +SELECT "TABLE1"."DIM1_1" "DIM1_1" + ,"TABLE2"."DIM2_1" "DIM2_1" + ,SUM("FACT"."M1") "M1" + ,SUM("FACT"."M2") "M2" + FROM ("COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1" + "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1") + LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2" + ON "FACT"."FK_2" = "TABLE2"."PK_2" + GROUP BY "TABLE2"."DIM2_1" + ,"TABLE1"."DIM1_1"; http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/test/resources/query/cognos/query01.sql.expected ---------------------------------------------------------------------- diff --git a/query/src/test/resources/query/cognos/query01.sql.expected b/query/src/test/resources/query/cognos/query01.sql.expected new file mode 100644 index 0000000..c8005fc --- /dev/null +++ b/query/src/test/resources/query/cognos/query01.sql.expected @@ -0,0 +1,28 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +SELECT "TABLE1"."DIM1_1" "DIM1_1" + ,"TABLE2"."DIM2_1" "DIM2_1" + ,SUM("FACT"."M1") "M1" + ,SUM("FACT"."M2") "M2" + FROM "COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1" + "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1" + LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2" + ON "FACT"."FK_2" = "TABLE2"."PK_2" + GROUP BY "TABLE2"."DIM2_1" + ,"TABLE1"."DIM1_1"; http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 6a38638..6fed9a8 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -65,6 +65,7 @@ import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.metadata.project.RealizationEntry; import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.query.util.QueryUtil; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.InternalErrorException; @@ -76,7 +77,6 @@ import org.apache.kylin.rest.model.TableMeta; import org.apache.kylin.rest.request.PrepareSqlRequest; import org.apache.kylin.rest.request.SQLRequest; import org.apache.kylin.rest.response.SQLResponse; -import org.apache.kylin.rest.util.QueryUtil; import org.apache.kylin.rest.util.Serializer; import org.apache.kylin.rest.util.TableauInterceptor; import org.apache.kylin.storage.hbase.HBaseConnection; @@ -447,7 +447,7 @@ public class QueryService extends BasicService { return fakeResponse; } - String correctedSql = QueryUtil.massageSql(sqlRequest); + String correctedSql = QueryUtil.massageSql(sqlRequest.getSql(), sqlRequest.getLimit(), sqlRequest.getOffset()); if (!correctedSql.equals(sqlRequest.getSql())) { logger.info("The corrected query: " + correctedSql); http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java deleted file mode 100644 index 33f4b63..0000000 --- a/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.rest.util; - -import java.util.LinkedList; -import java.util.List; -import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.rest.util.QueryUtil.IQueryTransformer; - -/** - * from (a join b on a.x = b.y) join c - * - * similar in https://issues.apache.org/jira/browse/CALCITE-35 - * - * we'll find such pattern and remove the parentheses - */ -public class CognosParenthesesEscape implements IQueryTransformer { - - private static final String S0 = "\\s*"; - private static final String S1 = "\\s"; - private static final String SM = "\\s+"; - private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+"; - private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS = TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?"; - private static final String JOIN = "(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server - private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME + S0 + "=" + S0 + TABLE_OR_COLUMN_NAME; - private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // ( - TABLE_NAME_WITH_OPTIONAL_ALIAS + // a - JOIN + SM + // join - TABLE_NAME_WITH_OPTIONAL_ALIAS + //b - SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*" + // on a.x = b.y [and a.x2 = b.y2] - S0 + "\\)";// ) - private static final Pattern PARENTTHESES_PATTERN = Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE); - - private static int identifierNum = 0; - - @Override - public String transform(String sql) { - Matcher m; - List<Pair<String, String>> matches = new LinkedList<>(); - while (true) { - m = PARENTTHESES_PATTERN.matcher(sql); - if (!m.find()) - break; - - String oneParentheses = m.group(0); - String identifier = generateRandomName(); - matches.add(new Pair<String, String>(identifier, oneParentheses.substring(1, oneParentheses.length() - 1))); - sql = sql.substring(0, m.start()) + identifier + sql.substring(m.end()); - } - - for (int i = matches.size() - 1; i >= 0; i--) { - sql = sql.replaceAll(matches.get(i).getKey(), matches.get(i).getValue()); - } - - return sql; - } - - private String generateRandomName() { - UUID uuid = UUID.randomUUID(); - return uuid.toString().replace("-", "_") + "_" + (identifierNum++); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java deleted file mode 100644 index d8c8845..0000000 --- a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.rest.util; - -import org.apache.kylin.rest.util.QueryUtil.IQueryTransformer; - -public class KeywordDefaultDirtyHack implements IQueryTransformer { - - @Override - public String transform(String sql) { - // KYLIN-2108, DEFAULT is hive default database, but a sql keyword too, needs quote - sql = sql.replace("DEFAULT.", "\"DEFAULT\"."); - sql = sql.replace("default.", "\"default\"."); - sql = sql.replace("defaultCatalog.", ""); - - return sql; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java deleted file mode 100644 index 26d6015..0000000 --- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.rest.util; - -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.rest.request.SQLRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -/** - */ -public class QueryUtil { - - protected static final Logger logger = LoggerFactory.getLogger(QueryUtil.class); - - private static List<IQueryTransformer> queryTransformers; - - public interface IQueryTransformer { - String transform(String sql); - } - - public static String massageSql(SQLRequest sqlRequest) { - String sql = sqlRequest.getSql(); - sql = sql.trim(); - sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator")); - - while (sql.endsWith(";")) - sql = sql.substring(0, sql.length() - 1); - - int limit = sqlRequest.getLimit(); - if (limit > 0 && !sql.toLowerCase().contains("limit")) { - sql += ("\nLIMIT " + limit); - } - - int offset = sqlRequest.getOffset(); - if (offset > 0 && !sql.toLowerCase().contains("offset")) { - sql += ("\nOFFSET " + offset); - } - - // customizable SQL transformation - if (queryTransformers == null) { - initQueryTransformers(); - } - for (IQueryTransformer t : queryTransformers) { - sql = t.transform(sql); - } - return sql; - } - - private static void initQueryTransformers() { - List<IQueryTransformer> transformers = Lists.newArrayList(); - transformers.add(new DefaultQueryTransformer()); - - String[] classes = KylinConfig.getInstanceFromEnv().getQueryTransformers(); - for (String clz : classes) { - try { - IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz); - transformers.add(t); - } catch (Exception e) { - logger.error("Failed to init query transformer", e); - } - } - queryTransformers = transformers; - } - - // correct sick / invalid SQL - private static class DefaultQueryTransformer implements IQueryTransformer { - - private static final String S0 = "\\s*"; - private static final String S1 = "\\s"; - private static final String SM = "\\s+"; - private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + "GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + "1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!="+ S0, Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_INTERVAL = Pattern.compile("interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + "(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE); - private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE); - - @Override - public String transform(String sql) { - Matcher m; - - // Case fn{ EXTRACT(...) } - // Use non-greedy regrex matching to remove escape functions - while (true) { - m = PTN_HAVING_ESCAPE_FUNCTION.matcher(sql); - if (!m.find()) - break; - sql = sql.substring(0, m.start()) + m.group(1) + sql.substring(m.end()); - } - - // Case: HAVING COUNT(1)>0 without Group By - // Tableau generates: SELECT SUM(1) AS "COL" FROM "VAC_SW" HAVING - // COUNT(1)>0 - m = PTN_HAVING_COUNT_GREATER_THAN_ZERO.matcher(sql); - if (m.find() && PTN_GROUP_BY.matcher(sql).find() == false) { - sql = sql.substring(0, m.start()) + " " + sql.substring(m.end()); - } - - // Case: SUM(1) - // Replace it with COUNT(1) - while (true) { - m = PTN_SUM_1.matcher(sql); - if (!m.find()) - break; - sql = sql.substring(0, m.start()) + " COUNT(1) " + sql.substring(m.end()); - } - - // Case: != - // Replace it with <> - while (true) { - m = PTN_NOT_EQ.matcher(sql); - if (!m.find()) - break; - sql = sql.substring(0, m.start()) + " <> " + sql.substring(m.end()); - } - - // ( date '2001-09-28' + interval floor(1) day ) generated by cognos - // calcite only recognizes date '2001-09-28' + interval '1' day - while (true) { - m = PTN_INTERVAL.matcher(sql); - if (!m.find()) - break; - - int value = (int) Math.floor(Double.valueOf(m.group(2))); - sql = sql.substring(0, m.start(1)) + "'" + value + "'" + sql.substring(m.end(3)); - } - - return sql; - } - - } - - public static String makeErrorMsgUserFriendly(Throwable e) { - String msg = e.getMessage(); - - // pick ParseException error message if possible - Throwable cause = e; - while (cause != null) { - if (cause.getClass().getName().contains("ParseException")) { - msg = cause.getMessage(); - break; - } - cause = cause.getCause(); - } - - return makeErrorMsgUserFriendly(msg); - } - - public static String makeErrorMsgUserFriendly(String errorMsg) { - try { - // make one line - errorMsg = errorMsg.replaceAll("\\s", " "); - - // move cause to be ahead of sql, calcite creates the message pattern below - Pattern pattern = Pattern.compile("error while executing SQL \"(.*)\":(.*)"); - Matcher matcher = pattern.matcher(errorMsg); - if (matcher.find()) { - return matcher.group(2).trim() + "\n" + "while executing SQL: \"" + matcher.group(1).trim() + "\""; - } else - return errorMsg; - } catch (Exception e) { - return errorMsg; - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java ---------------------------------------------------------------------- diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java deleted file mode 100644 index 76df219..0000000 --- a/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kylin.rest.util; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.Collection; - -import org.apache.commons.io.FileUtils; -import org.junit.Assert; -import org.junit.Test; - -public class CognosParentesesEscapeTest { - - @Test - public void basicTest() { - CognosParenthesesEscape escape = new CognosParenthesesEscape(); - String data = "((a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3) inner join c as cc on a.x1=cc.z1 ) join d dd on a.x1=d.w1 and a.x2 =d.w2 "; - String expected = "a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3 inner join c as cc on a.x1=cc.z1 join d dd on a.x1=d.w1 and a.x2 =d.w2 "; - String transformed = escape.transform(data); - Assert.assertEquals(expected, transformed); - } - - @Test - public void advancedTest() throws IOException { - CognosParenthesesEscape escape = new CognosParenthesesEscape(); - String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql"), Charset.defaultCharset()); - String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql.expected"), Charset.defaultCharset()); - String transformed = escape.transform(query); - //System.out.println(transformed); - Assert.assertEquals(expected, transformed); - } - - @Test - public void proguardTest() throws IOException { - CognosParenthesesEscape escape = new CognosParenthesesEscape(); - Collection<File> files = FileUtils.listFiles(new File("../kylin-it/src/test/resources"), new String[] { "sql" }, true); - for (File f : files) { - System.out.println("checking " + f.getAbsolutePath()); - String query = FileUtils.readFileToString(f, Charset.defaultCharset()); - String transformed = escape.transform(query); - Assert.assertEquals(query, transformed); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java ---------------------------------------------------------------------- diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java deleted file mode 100644 index dcd8c98..0000000 --- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.rest.util; - -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.rest.request.SQLRequest; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class QueryUtilTest extends LocalFileMetadataTestCase { - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testMassageSql() { - { - SQLRequest sqlRequest = new SQLRequest(); - sqlRequest.setSql("select ( date '2001-09-28' + interval floor(1.2) day) from test_kylin_fact"); - String s = QueryUtil.massageSql(sqlRequest); - Assert.assertEquals("select ( date '2001-09-28' + interval '1' day) from test_kylin_fact", s); - } - { - SQLRequest sqlRequest = new SQLRequest(); - sqlRequest.setSql("select ( date '2001-09-28' + interval floor(2) month) from test_kylin_fact group by ( date '2001-09-28' + interval floor(2) month)"); - String s = QueryUtil.massageSql(sqlRequest); - Assert.assertEquals("select ( date '2001-09-28' + interval '2' month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' month)", s); - } - } - - @Test - public void testKeywordDefaultDirtyHack() { - { - SQLRequest sqlRequest = new SQLRequest(); - sqlRequest.setSql("select * from DEFAULT.TEST_KYLIN_FACT"); - String s = QueryUtil.massageSql(sqlRequest); - Assert.assertEquals("select * from \"DEFAULT\".TEST_KYLIN_FACT", s); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/test/resources/query/cognos/query01.sql ---------------------------------------------------------------------- diff --git a/server-base/src/test/resources/query/cognos/query01.sql b/server-base/src/test/resources/query/cognos/query01.sql deleted file mode 100644 index a76b4e1..0000000 --- a/server-base/src/test/resources/query/cognos/query01.sql +++ /dev/null @@ -1,28 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -SELECT "TABLE1"."DIM1_1" "DIM1_1" - ,"TABLE2"."DIM2_1" "DIM2_1" - ,SUM("FACT"."M1") "M1" - ,SUM("FACT"."M2") "M2" - FROM ("COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1" - "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1") - LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2" - ON "FACT"."FK_2" = "TABLE2"."PK_2" - GROUP BY "TABLE2"."DIM2_1" - ,"TABLE1"."DIM1_1"; http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/test/resources/query/cognos/query01.sql.expected ---------------------------------------------------------------------- diff --git a/server-base/src/test/resources/query/cognos/query01.sql.expected b/server-base/src/test/resources/query/cognos/query01.sql.expected deleted file mode 100644 index c8005fc..0000000 --- a/server-base/src/test/resources/query/cognos/query01.sql.expected +++ /dev/null @@ -1,28 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -SELECT "TABLE1"."DIM1_1" "DIM1_1" - ,"TABLE2"."DIM2_1" "DIM2_1" - ,SUM("FACT"."M1") "M1" - ,SUM("FACT"."M2") "M2" - FROM "COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1" - "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1" - LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2" - ON "FACT"."FK_2" = "TABLE2"."PK_2" - GROUP BY "TABLE2"."DIM2_1" - ,"TABLE1"."DIM1_1"; http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java index e84235b..2043fef 100644 --- a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java +++ b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java @@ -19,12 +19,12 @@ package org.apache.kylin.rest.controller; import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.query.util.QueryUtil; import org.apache.kylin.rest.request.MetaRequest; import org.apache.kylin.rest.request.SQLRequest; import org.apache.kylin.rest.response.SQLResponse; import org.apache.kylin.rest.service.QueryService; import org.apache.kylin.rest.service.ServiceTestBase; -import org.apache.kylin.rest.util.QueryUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test;
