if outer query does not have aggr, join columns with subquery should not be added to groupbycolumns (squashed)
fix CI fix CI bug fix: if outer query does not have aggr, join columns with subquery should not be added to groupbycolumns bug fix Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8acf14f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8acf14f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8acf14f Branch: refs/heads/master Commit: b8acf14f5e963e48024cd479b64dcf9f1be4d2e0 Parents: a0c4bfb Author: Hongbin Ma <[email protected]> Authored: Thu Nov 3 18:32:19 2016 +0800 Committer: Hongbin Ma <[email protected]> Committed: Mon Nov 7 15:01:15 2016 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/cube/cuboid/Cuboid.java | 5 +++++ .../apache/kylin/metadata/realization/SQLDigest.java | 13 ++++++++++++- .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java | 3 +++ .../org/apache/kylin/storage/hbase/ITStorageTest.java | 4 +++- .../org/apache/kylin/query/relnode/OLAPContext.java | 7 +++---- .../org/apache/kylin/query/relnode/OLAPJoinRel.java | 5 ++--- .../kylin/storage/hbase/cube/v1/CubeStorageQuery.java | 4 ++++ 7 files changed, 32 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java index 855aad6..7503fbf 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java @@ -89,6 +89,11 @@ public class Cuboid implements Comparable<Cuboid> { } public static boolean isValid(CubeDesc cube, long cuboidID) { + //base cuboid is always valid + if (cuboidID == getBaseCuboidId(cube)) { + return true; + } + for (AggregationGroup agg : cube.getAggregationGroups()) { if (isValid(agg, cuboidID)) { return true; http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java index 90fa53a..83fc05c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java @@ -51,6 +51,7 @@ public class SQLDigest { public List<JoinDesc> joinDescs; public Set<TblColRef> allColumns; public List<TblColRef> groupbyColumns; + public Set<TblColRef> subqueryJoinParticipants; public Set<TblColRef> filterColumns; public Set<TblColRef> metricColumns; public List<FunctionDesc> aggregations; // storage level measure type, on top of which various sql aggr function may apply @@ -60,13 +61,14 @@ public class SQLDigest { public boolean isRawQuery; public SQLDigest(String factTable, TupleFilter filter, List<JoinDesc> joinDescs, Set<TblColRef> allColumns, // - List<TblColRef> groupbyColumns, Set<TblColRef> filterColumns, Set<TblColRef> metricColumns, // + List<TblColRef> groupbyColumns, Set<TblColRef> subqueryJoinParticipants, Set<TblColRef> filterColumns, Set<TblColRef> metricColumns, // List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef> sortColumns, List<OrderEnum> sortOrders) { this.factTable = factTable; this.filter = filter; this.joinDescs = joinDescs; this.allColumns = allColumns; this.groupbyColumns = groupbyColumns; + this.subqueryJoinParticipants = subqueryJoinParticipants; this.filterColumns = filterColumns; this.metricColumns = metricColumns; this.aggregations = aggregations; @@ -82,6 +84,15 @@ public class SQLDigest { //the reason to choose aggregations rather than metricColumns is because the former is set earlier at implOLAP } + public void includeSubqueryJoinParticipants() { + if (this.isRawQuery) { + this.allColumns.addAll(this.subqueryJoinParticipants); + } else { + this.groupbyColumns.addAll(this.subqueryJoinParticipants); + this.allColumns.addAll(this.subqueryJoinParticipants); + } + } + @Override public String toString() { return "fact table " + this.factTable + "," + // http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index b96e952..b51af59 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -74,6 +74,9 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { @Override public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { + //deal with participant columns in subquery join + sqlDigest.includeSubqueryJoinParticipants(); + //cope with queries with no aggregations RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc); http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java index ea3cbdb..78b87fd 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java @@ -48,6 +48,8 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import com.google.common.collect.Sets; + public class ITStorageTest extends HBaseMetadataTestCase { private IStorageQuery storageEngine; @@ -146,7 +148,7 @@ public class ITStorageTest extends HBaseMetadataTestCase { int count = 0; ITupleIterator iterator = null; try { - SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>()); + SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>()); iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations)); while (iterator.hasNext()) { ITuple tuple = iterator.next(); http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java index d7f68ac..615d802 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java @@ -120,6 +120,7 @@ public class OLAPContext { public Set<TblColRef> allColumns = new HashSet<>(); public List<TblColRef> groupByColumns = new ArrayList<>(); + public Set<TblColRef> subqueryJoinParticipants = new HashSet<TblColRef>();//subqueryJoinParticipants will be added to groupByColumns(only when other group by co-exists) and allColumns public Set<TblColRef> metricsColumns = new HashSet<>(); public List<FunctionDesc> aggregations = new ArrayList<>(); // storage level measure type, on top of which various sql aggr function may apply public List<SQLCall> aggrSqlCalls = new ArrayList<>(); // sql level aggregation function call @@ -147,7 +148,7 @@ public class OLAPContext { public SQLDigest getSQLDigest() { if (sqlDigest == null) - sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders); + sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders); return sqlDigest; } @@ -176,10 +177,8 @@ public class OLAPContext { sortOrders.add(order); } } - + // ============================================================================ - - public interface IAccessController { /* http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java index e75d747..265d65d 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java @@ -48,9 +48,9 @@ import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.RelDataTypeFieldImpl; -import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; @@ -167,8 +167,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { for (Map.Entry<TblColRef, TblColRef> columnPair : joinCol.entrySet()) { TblColRef fromCol = (rightHasSubquery ? columnPair.getKey() : columnPair.getValue()); - this.context.groupByColumns.add(fromCol); - this.context.allColumns.add(fromCol); + this.context.subqueryJoinParticipants.add(fromCol); } joinCol.clear(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b8acf14f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index ff729f4..3c73dc6 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -98,6 +98,10 @@ public class CubeStorageQuery implements IStorageQuery { @Override public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { + + //deal with participant columns in subquery join + sqlDigest.includeSubqueryJoinParticipants(); + //cope with queries with no aggregations RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
