Repository: kylin Updated Branches: refs/heads/master b109948e0 -> 1d4c7bb82
KYLIN-1561 remaing part of b26b2489baddcfd148eaf8f17330878bbf349048 Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d4c7bb8 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d4c7bb8 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d4c7bb8 Branch: refs/heads/master Commit: 1d4c7bb82059176aa8fb0f2805675f64ad603412 Parents: b109948 Author: Hongbin Ma <[email protected]> Authored: Wed Apr 13 17:16:00 2016 +0800 Committer: Hongbin Ma <[email protected]> Committed: Wed Apr 13 17:16:06 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 13 +++++++- .../org/apache/kylin/query/KylinTestBase.java | 4 ++- .../src/test/resources/query/sql/query99.sql | 33 ++++++++++++++++++++ .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 2 +- .../hbase/cube/v2/CubeSegmentScanner.java | 9 +++++- 5 files changed, 57 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index f50793e..286c20c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -499,6 +499,18 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", "false")); } + public String getQueryStorageVisitPlanner() { + return this.getOptional("kylin.query.storage.visit.planner", "org.apache.kylin.gridtable.GTScanRangePlanner"); + } + + public void setQueryStorageVisitPlanner(String v) { + setProperty("kylin.query.storage.visit.planner", v); + } + + public int getQueryStorageVisitScanRangeMax() { + return Integer.valueOf(this.getOptional("kylin.query.storage.visit.scanrange.max", "1000000")); + } + public int getHBaseKeyValueSize() { return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760")); } @@ -639,5 +651,4 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.cube.mr.engine.v2.class", "org.apache.kylin.engine.mr.MRBatchCubingEngine2"); } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index 46aa68d..d3c8f6d 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -84,6 +84,7 @@ public class KylinTestBase { protected static int compQueryCount = 0; protected static ArrayList<String> zeroResultQueries = new ArrayList<String>(); + protected static String ITDirHeader = ""; protected static void closeConnection(Connection connection) { if (connection != null) { @@ -108,6 +109,7 @@ public class KylinTestBase { * @return */ protected static List<File> getFilesFromFolder(final File folder, final String fileType) { + System.out.println(folder.getAbsolutePath()); Set<File> set = new TreeSet<>(new FileByNameComparator()); for (final File fileEntry : folder.listFiles()) { if (fileEntry.getName().toLowerCase().endsWith(fileType.toLowerCase())) { @@ -429,7 +431,7 @@ public class KylinTestBase { protected int runSQL(File sqlFile, boolean debug, boolean explain) throws Exception { if (debug) { System.setProperty("calcite.debug", "true"); - InputStream inputStream = new FileInputStream("src/test/resources/logging.properties"); + InputStream inputStream = new FileInputStream(ITDirHeader + "src/test/resources/logging.properties"); LogManager.getLogManager().readConfiguration(inputStream); } http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/kylin-it/src/test/resources/query/sql/query99.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql/query99.sql b/kylin-it/src/test/resources/query/sql/query99.sql new file mode 100644 index 0000000..80a537d --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query99.sql @@ -0,0 +1,33 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +SELECT + test_kylin_fact.seller_id + ,test_category_groupings.meta_categ_name + ,test_kylin_fact.lstg_format_name + ,sum(price) as GMV, count(*) as TRANS_CNT + FROM test_kylin_fact + inner JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + inner JOIN test_category_groupings + ON test_kylin_fact.lstg_site_id = test_category_groupings.site_id AND test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id + where test_category_groupings.meta_categ_name='Collectibles' + group by test_kylin_fact.seller_id + ,test_category_groupings.meta_categ_name + ,test_kylin_fact.lstg_format_name + http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index 81d5baa..d86b37f 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -317,7 +317,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { logger.debug("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size()); - logger.info("The scan {} for segment {} is as below, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg); + logger.info("The scan {} for segment {} is as below with {} separate raw scans, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg,rawScans.size()); for (RawScan rs : rawScans) { logScan(rs, cubeSeg.getStorageLocationIdentifier()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java index 9ed914a..2b55ace 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.Set; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.dict.BuildInFunctionTransformer; @@ -58,7 +59,13 @@ public class CubeSegmentScanner implements IGTScanner { ITupleFilterTransformer translator = new BuildInFunctionTransformer(cubeSeg.getDimensionEncodingMap()); filter = translator.transform(filter); - GTScanRangePlanner scanRangePlanner = new GTScanRangePlanner(cubeSeg, cuboid, filter, dimensions, groups, metrics); + String plannerName = KylinConfig.getInstanceFromEnv().getQueryStorageVisitPlanner(); + GTScanRangePlanner scanRangePlanner; + try { + scanRangePlanner = (GTScanRangePlanner) Class.forName(plannerName).getConstructor(CubeSegment.class, Cuboid.class, TupleFilter.class, Set.class, Set.class, Collection.class).newInstance(cubeSeg, cuboid, filter, dimensions, groups, metrics); + } catch (Exception e) { + throw new RuntimeException(e); + } scanRequest = scanRangePlanner.planScanRequest(allowPreAggregate); scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest); }
