Repository: kylin Updated Branches: refs/heads/yang21 698b542a1 -> d7657ff39
KYLIN-2277 return correct columns for select * Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7657ff3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7657ff3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7657ff3 Branch: refs/heads/yang21 Commit: d7657ff3916605e83e67596e57e6d5e197dd3624 Parents: 698b542 Author: Yang Li <[email protected]> Authored: Tue Dec 13 23:45:55 2016 +0800 Committer: Yang Li <[email protected]> Committed: Wed Dec 14 21:26:49 2016 +0800 ---------------------------------------------------------------------- .../calcite/sql2rel/SqlToRelConverter.java | 62 +++++++++++++++++++- .../kylin/metadata/model/FunctionDesc.java | 4 +- .../apache/kylin/query/ITKylinQueryTest.java | 26 ++++++++ .../org/apache/kylin/query/KylinTestBase.java | 21 +++++++ .../resources/query/sql_timeout/query01.sql | 2 +- 5 files changed, 111 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d7657ff3/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java ---------------------------------------------------------------------- diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index d223cdf..cf36f61 100644 --- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -67,6 +67,7 @@ import org.apache.calcite.rel.stream.Delta; import org.apache.calcite.rel.stream.LogicalDelta; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -194,6 +195,7 @@ import static org.apache.calcite.util.Static.RESOURCE; * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE` * - isTrimUnusedFields(), override to false * - AggConverter.translateAgg(...), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104 + * - convertQuery(), call hackSelectStar() at the end */ /** @@ -529,6 +531,8 @@ public class SqlToRelConverter { * the query will be part of a view. */ public RelRoot convertQuery(SqlNode query, final boolean needsValidation, final boolean top) { + SqlNode origQuery = query; /* OVERRIDE POINT */ + if (needsValidation) { query = validator.validate(query); } @@ -553,7 +557,63 @@ public class SqlToRelConverter { } final RelDataType validatedRowType = validator.getValidatedNodeType(query); - return RelRoot.of(result, validatedRowType, query.getKind()).withCollation(collation); + return hackSelectStar(origQuery, RelRoot.of(result, validatedRowType, query.getKind()).withCollation(collation)); + } + + /* OVERRIDE POINT */ + private RelRoot hackSelectStar(SqlNode query, RelRoot root) { + /* + * Rel tree is like: + * + * LogicalSort (optional) + * |- LogicalProject + * |- OLAPTableScan + */ + LogicalProject rootPrj = null; + LogicalSort rootSort = null; + if (root.rel instanceof LogicalProject) { + rootPrj = (LogicalProject) root.rel; + } else if (root.rel instanceof LogicalSort && root.rel.getInput(0) instanceof LogicalProject) { + rootPrj = (LogicalProject) root.rel.getInput(0); + rootSort = (LogicalSort) root.rel; + } else { + return root; + } + + if (!rootPrj.getInput().getClass().getSimpleName().equals("OLAPTableScan")) + return root; + + RelNode scan = rootPrj.getInput(); + if (rootPrj.getRowType().getFieldCount() < scan.getRowType().getFieldCount()) + return root; + + RelDataType inType = rootPrj.getRowType(); + List<String> inFields = inType.getFieldNames(); + List<RexNode> projExp = new ArrayList<>(); + List<Pair<Integer, String>> projFields = new ArrayList<>(); + FieldInfoBuilder projTypeBuilder = getCluster().getTypeFactory().builder(); + FieldInfoBuilder validTypeBuilder = getCluster().getTypeFactory().builder(); + for (int i = 0; i < inFields.size(); i++) { + if (!inFields.get(i).startsWith("_KY_")) { + projExp.add(rootPrj.getProjects().get(i)); + projFields.add(Pair.of(projFields.size(), inFields.get(i))); + projTypeBuilder.add(inType.getFieldList().get(i)); + validTypeBuilder.add(root.validatedRowType.getFieldList().get(i)); + } + } + + RelDataType projRowType = getCluster().getTypeFactory().createStructType(projTypeBuilder); + rootPrj = LogicalProject.create(scan, projExp, projRowType); + if (rootSort != null) { + rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch); + } + + RelDataType validRowType = getCluster().getTypeFactory().createStructType(validTypeBuilder); + root = new RelRoot(rootSort == null ? rootPrj : rootSort, validRowType, root.kind, projFields, root.collation); + + validator.setValidatedNodeType(query, validRowType); + + return root; } private static boolean isStream(SqlNode query) { http://git-wip-us.apache.org/repos/asf/kylin/blob/d7657ff3/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index 0946703..6d3dc92 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -148,9 +148,9 @@ public class FunctionDesc { if (isSum()) { return getParameter().getValue(); } else if (isCount()) { - return "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same + return "_KY_" + "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same } else { - return getFullExpression().replaceAll("[(), ]", "_"); + return "_KY_" + getFullExpression().replaceAll("[(),. ]", "_"); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/d7657ff3/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 0740ffa..a9d77e5 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.KylinVersion; import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.gridtable.GTScanSelfTerminatedException; import org.apache.kylin.gridtable.StorageSideBehavior; @@ -36,7 +37,9 @@ import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; import org.apache.kylin.storage.hbase.HBaseStorage; import org.dbunit.database.DatabaseConnection; import org.dbunit.database.IDatabaseConnection; +import org.dbunit.dataset.ITable; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Rule; @@ -356,4 +359,27 @@ public class ITKylinQueryTest extends KylinTestBase { this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_window"); } + @Test + public void testVersionQuery() throws Exception { + String expectVersion = KylinVersion.getCurrentVersion().toString(); + printInfo("---------- verify expect version: " + expectVersion); + + String queryName = "QueryKylinVersion"; + String sql = "SELECT VERSION() AS version"; + + // execute Kylin + printInfo("Query Result from Kylin - " + queryName); + IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection); + ITable kylinTable = executeQuery(kylinConn, queryName, sql, false); + String queriedVersion = String.valueOf(kylinTable.getValue(0, "version")); + + // compare the result + Assert.assertEquals(expectVersion, queriedVersion); + } + + @Test + public void testSelectStarColumnCount() throws Exception { + execAndCompColumnCount("select * from test_kylin_fact limit 10", 9); + execAndCompColumnCount("select * from test_kylin_fact", 9); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/d7657ff3/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index d0bcf52..feed135 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -66,6 +66,7 @@ import org.dbunit.ext.h2.H2Connection; import org.dbunit.ext.h2.H2DataTypeFactory; import org.junit.Assert; +import com.google.common.collect.ImmutableSet; import com.google.common.io.Files; /** @@ -398,6 +399,26 @@ public class KylinTestBase { } } + protected void execAndCompColumnCount(String input, int expectedColumnCount) throws Exception { + printInfo("---------- test column count: " + input); + Set<String> sqlSet = ImmutableSet.of(input); + + for (String sql : sqlSet) { + // execute Kylin + printInfo("Query Result from Kylin - " + sql); + IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection); + ITable kylinTable = executeQuery(kylinConn, sql, sql, false); + + try { + // compare the result + Assert.assertEquals(expectedColumnCount, kylinTable.getTableMetaData().getColumns().length); + } catch (Throwable t) { + printInfo("execAndCompColumnCount failed on: " + sql); + throw t; + } + } + } + protected void execLimitAndValidate(String queryFolder) throws Exception { printInfo("---------- test folder: " + new File(queryFolder).getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/kylin/blob/d7657ff3/kylin-it/src/test/resources/query/sql_timeout/query01.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_timeout/query01.sql b/kylin-it/src/test/resources/query/sql_timeout/query01.sql index 3b9a837..eaff396 100644 --- a/kylin-it/src/test/resources/query/sql_timeout/query01.sql +++ b/kylin-it/src/test/resources/query/sql_timeout/query01.sql @@ -16,4 +16,4 @@ -- limitations under the License. -- -select * from test_kylin_fact limit 1200 +select seller_id,lstg_format_name from test_kylin_fact limit 1200
