[kylin] branch kylin-on-druid updated: Add patch for precise-distinct-count, extend-column, decima measure
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch kylin-on-druid in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/kylin-on-druid by this push: new 4b6668a Add patch for precise-distinct-count,extend-column,decima measure 4b6668a is described below commit 4b6668af91c14a318b3e06a7ac8deef91a1b326b Author: kangkaisen AuthorDate: Thu Dec 27 20:43:51 2018 +0800 Add patch for precise-distinct-count,extend-column,decima measure --- ...tinct-count-extend-column-decimal-measure.patch | 3960 1 file changed, 3960 insertions(+) diff --git a/kod-precise-distinct-count-extend-column-decimal-measure.patch b/kod-precise-distinct-count-extend-column-decimal-measure.patch new file mode 100644 index 000..b7a0539 --- /dev/null +++ b/kod-precise-distinct-count-extend-column-decimal-measure.patch @@ -0,0 +1,3960 @@ +From 4e7daa032d5e6867666b65843e2804ffe1de2c67 Mon Sep 17 00:00:00 2001 +From: kangkaisen +Date: Wed, 15 Nov 2017 19:30:54 +0800 +Subject: [PATCH] Druid support kylin precise distinct count measure Druid + support kylin extend-column measure Druid support decimal measure + +--- + .../bitmap/WrappedImmutableRoaringBitmap.java | 2 + + distribution/pom.xml | 12 + + extensions-contrib/decimal/pom.xml| 59 + + .../decimal/DecimalAggregatorFactory.java | 114 + .../decimal/DecimalBufferAggregator.java | 99 +++ + .../decimal/DecimalDruidModule.java | 71 + + .../decimal/DecimalMaxAggregator.java | 82 ++ + .../decimal/DecimalMaxAggregatorFactory.java | 160 +++ + .../decimal/DecimalMaxBufferAggregator.java | 56 + .../aggregation/decimal/DecimalMaxSerde.java | 29 ++ + .../decimal/DecimalMinAggregator.java | 83 ++ + .../decimal/DecimalMinAggregatorFactory.java | 149 +++ + .../decimal/DecimalMinBufferAggregator.java | 56 + .../aggregation/decimal/DecimalMinSerde.java | 29 ++ + .../aggregation/decimal/DecimalSerde.java | 135 ++ + .../decimal/DecimalSumAggregator.java | 80 ++ + .../decimal/DecimalSumAggregatorFactory.java | 143 ++ + .../decimal/DecimalSumBufferAggregator.java | 54 + .../aggregation/decimal/DecimalSumSerde.java | 29 ++ + .../io.druid.initialization.DruidModule | 1 + + .../DecimalGroupByQueryTest.java | 250 ++ + extensions-contrib/kylin-distinccount/pom.xml | 59 + + .../DistinctCountAggregator.java | 73 + + .../DistinctCountAggregatorFactory.java | 223 + .../DistinctCountBufferAggregator.java| 120 + + .../DistinctCountDruidModule.java | 53 + .../DistinctCountSerde.java | 95 +++ + .../io.druid.initialization.DruidModule | 1 + + .../DistinctCountBufferAggregatorTest.java| 71 + + .../DistinctCountGroupByQueryTest.java| 234 + extensions-contrib/kylin-extendcolumn/pom.xml | 59 + + .../ExtendColumnAggregator.java | 83 ++ + .../ExtendColumnAggregatorFactory.java| 227 + .../ExtendColumnBufferAggregator.java | 117 + .../ExtendColumnDruidModule.java | 51 + .../kylin.extendcolumn/ExtendColumnSerde.java | 139 ++ + .../io.druid.initialization.DruidModule | 1 + + .../ExtendColumnGroupByQueryTest.java | 229 + pom.xml | 3 + + .../DruidDefaultSerializersModule.java| 18 ++ + .../io/druid/segment/IndexMergerTestBase.java | 2 +- + .../java/io/druid/segment/TestHelper.java | 9 +- + 42 files changed, 3558 insertions(+), 2 deletions(-) + create mode 100644 extensions-contrib/decimal/pom.xml + create mode 100644 extensions-contrib/decimal/src/main/java/io/druid/query/aggregation/decimal/DecimalAggregatorFactory.java + create mode 100644 extensions-contrib/decimal/src/main/java/io/druid/query/aggregation/decimal/DecimalBufferAggregator.java + create mode 100644 extensions-contrib/decimal/src/main/java/io/druid/query/aggregation/decimal/DecimalDruidModule.java + create mode 100644 extensions-contrib/decimal/src/main/java/io/druid/query/aggregation/decimal/DecimalMaxAggregator.java + create mode 100644 extensions-contrib/decimal/src/main/java/io/druid/query/aggregation/decimal/DecimalMaxAggregatorFactory.java + create mode 100644 extensions-contrib/decimal/src/main/java/io/druid/query/aggregation/decimal/DecimalMaxBufferAggregator.java + create mode 100644 extensions-contrib/decimal/src/main/java/io/druid/query/aggregation/decimal/DecimalMaxSerde.java + create mode 100644 extensions-contrib/decimal/src/main/java/io/druid/query/aggregation/decimal/DecimalMinAggregator.java + create mode 100644 extensions-contrib/decimal/src/main/java
[kylin] branch kylin-on-druid updated: Update PruneIntervalsProcessor.java (#419)
This is an automated email from the ASF dual-hosted git repository.
kangkaisen pushed a commit to branch kylin-on-druid
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-druid by this push:
new 6e79781 Update PruneIntervalsProcessor.java (#419)
6e79781 is described below
commit 6e79781d825a9e6fdab837e4d928bf7ddbe0bee9
Author: 球球爸爸
AuthorDate: Thu Dec 27 10:34:42 2018 +0800
Update PruneIntervalsProcessor.java (#419)
---
.../apache/kylin/storage/druid/read/filter/PruneIntervalsProcessor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/filter/PruneIntervalsProcessor.java
b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/filter/PruneIntervalsProcessor.java
index d19e9cd..0b30452 100644
---
a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/filter/PruneIntervalsProcessor.java
+++
b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/filter/PruneIntervalsProcessor.java
@@ -110,7 +110,7 @@ public class PruneIntervalsProcessor implements
Function segmentRanges = TreeRangeSet.create();
for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
-Range segmentRange =
Range.closedOpen(segment.getTSRange().start.v, segment.getTSRange().start.v);
+Range segmentRange =
Range.closedOpen(segment.getTSRange().start.v, segment.getTSRange().end.v);
boolean containQueryRange =
!result.timeRanges.subRangeSet(segmentRange).isEmpty();
if (segment.getInputRecords() > 0 && containQueryRange) {
segmentRanges.add(segmentRange);
[kylin] branch kylin-on-druid created (now 31cacb5)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to branch kylin-on-druid in repository https://gitbox.apache.org/repos/asf/kylin.git. at 31cacb5 Kylin-on-Druid Storage This branch includes the following new commits: new 31cacb5 Kylin-on-Druid Storage The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[kylin] branch document updated: KYLIN-3425 Update release notes and download for 2.3.2 2
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch document in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/document by this push: new ca79631 KYLIN-3425 Update release notes and download for 2.3.2 2 ca79631 is described below commit ca79631aec0b5682df012fdc9e12d7bbdb78d8ca Author: kangkaisen AuthorDate: Sun Jul 8 15:15:36 2018 +0800 KYLIN-3425 Update release notes and download for 2.3.2 2 --- website/download/index.cn.md | 2 +- website/download/index.md| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/website/download/index.cn.md b/website/download/index.cn.md index 438022c..9d64c33 100644 --- a/website/download/index.cn.md +++ b/website/download/index.cn.md @@ -19,7 +19,7 @@ title: 下载 - 源代码下载: [apache-kylin-2.3.2-src.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip.asc)\] \[[sha1](https://www.apache.org/dist/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip.sha1)\] - 二进制包下载: - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - [apache-kylin-2.3.2-bin-hbase1x.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-hbase1x.tar.gz) \[[asc](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-hbase1x.tar.gz.asc)\] \[[md5](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-hbase1x.tar.gz.md5)\] - - for CDH 5.7+ - [apache-kylin-2.3.1-bin-cdh57.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz) \[[asc](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.asc)\] \[[md5](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.md5)\] + - for CDH 5.7+ - [apache-kylin-2.3.2-bin-cdh57.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz) \[[asc](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.asc)\] \[[md5](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.md5)\] JDBC 驱动程序 diff --git a/website/download/index.md b/website/download/index.md index 2489b29..2862f62 100644 --- a/website/download/index.md +++ b/website/download/index.md @@ -21,7 +21,7 @@ You can verify the download by following these [procedures](https://www.apache.o - Source download: [apache-kylin-2.3.2-src.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip.asc)\] \[[sha1](https://www.apache.org/dist/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip.sha1)\] - Binary download: - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - [apache-kylin-2.3.2-bin-hbase1x.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-hbase1x.tar.gz) \[[asc](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-hbase1x.tar.gz.asc)\] \[[md5](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-hbase1x.tar.gz.md5)\] - - for CDH 5.7+ - [apache-kylin-2.3.1-bin-cdh57.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz) \[[asc](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.asc)\] \[[md5](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.md5)\] + - for CDH 5.7+ - [apache-kylin-2.3.2-bin-cdh57.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz) \[[asc](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.asc)\] \[[md5](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.md5)\] JDBC Driver
svn commit: r1835341 - in /kylin/site: cn/download/index.html docs23/release_notes.html download/index.html
Author: kangkaisen Date: Sun Jul 8 07:34:45 2018 New Revision: 1835341 URL: http://svn.apache.org/viewvc?rev=1835341&view=rev Log: KYLIN-3425 Update release notes and download for 2.3.2 Modified: kylin/site/cn/download/index.html kylin/site/docs23/release_notes.html kylin/site/download/index.html Modified: kylin/site/cn/download/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn/download/index.html?rev=1835341&r1=1835340&r2=1835341&view=diff == --- kylin/site/cn/download/index.html (original) +++ kylin/site/cn/download/index.html Sun Jul 8 07:34:45 2018 @@ -187,31 +187,31 @@ var _hmt = _hmt || []; -v2.3.1 - +v2.3.2 - Release notes å å级æå - æºä»£ç ä¸è½½: http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-src.tar.gz";>apache-kylin-2.3.1-src.tar.gz [https://www.apache.org/dist/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-src.tar.gz.asc";>asc] [https://www.apache.org/dist/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-src.tar.gz.sha1";>sha1] + This is a bug fix release after 2.3.1, with 12 bug fixes and enhancements. For the detail list please check release notes. + Release notes and å级æå + æºä»£ç ä¸è½½: https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip";>apache-kylin-2.3.2-src.tar.gz [https://www.apache.org/dist/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip.asc";>asc] [https://www.apache.org/dist/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip.sha1";>sha1] äºè¿å¶å ä¸è½½: - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-hbase1x-bin.tar.gz";>apache-kylin-2.3.1-bin-hbase1x.tar.gz [https://www.apache.org/dist/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-hbase1x-bin.tar.gz.asc";>asc] [https://www.apache.org/dist/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-hbase1x-bin.tar.gz.sha1";>sha1] - for CDH 5.7+ - http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-cdh57-bin.tar.gz";>apache-kylin-2.3.1-bin-cdh57.tar.gz [https://www.apache.org/dist/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-cdh57-bin.tar.gz.asc";>asc] [https://www.apache.org/dist/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-cdh57-bin.tar.gz.sha1";>sha1] + for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-hbase1x.tar.gz";>apache-kylin-2.3.2-bin-hbase1x.tar.gz [https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-hbase1x.tar.gz.asc";>asc] [https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-hbase1x.tar.gz.md5";>md5] + for CDH 5.7+ - https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz";>apache-kylin-2.3.2-bin-cdh57.tar.gz [https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.asc";>asc] [https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-bin-cdh57.tar.gz.md5";>md5] -JDBC 驱å¨ç¨åº +JDBC 驱å¨ç¨åº Kylin JDBC 驱å¨ç¨åº, kylin-jdbc-<version>.jar, å¨ Kylin äºè¿å¶å ç âlibâ ç®å½ä¸. -ODBC 驱å¨ç¨åº +ODBC 驱å¨ç¨åº http://kylin.apache.org/download/KylinODBCDriver-2.1.0.zip";>Kylin ODBC é©±å¨ v2.1.0 ï¼ä¸ææ Kylin çæ¬å ¼å®¹ï¼ 注: Kylin ODBC 驱å¨ä¾èµ http://www.microsoft.com/en-us/download/details.aspx?id=30679";>Microsoft Visual C++ 2012 Redistributableã -以åççæ¬ +以åççæ¬ Apache Kylinçæ§çæ¬å¯ä»¥ä» https://archive.apache.org/dist/kylin/";>https://archive.apache.org/dist/kylin/ ä¸è½½ã Modified: kylin/site/docs23/release_notes.html URL: http://svn.apache.org/viewvc/kylin/site/docs23/release_notes.html?rev=1835341&r1=1835340&r2=1835341&view=diff == --- kylin/site/docs23/release_notes.html (original) +++ kylin/site/docs23/release_notes.html Sun Jul 8 07:34:45 2018 @@ -5404,380 +5404,428 @@ there are source code package, binary pa or send to Apache Kylin mailing list: - User relative: user@kylin.apache.org - Development relative: dev@kylin.apache.org + User relative: mailto:u...@kylin.apache.org";>u...@kylin.apache.org + Development relative: mailto:d...@kylin.apache.org";>d...@kylin.apache.org + + +v2.3.2 - 2018-07-08 +Tag: https://github.com/apache/kylin/tree/kylin-2.3.2";>kylin-2.3.2 +This is a bug fix rele
[kylin] branch document updated: KYLIN-3425 Update release notes and download for 2.3.2
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch document in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/document by this push: new e7a40ab KYLIN-3425 Update release notes and download for 2.3.2 e7a40ab is described below commit e7a40ab3ac6dab784c957ef3131e9f7e07e40277 Author: kangkaisen AuthorDate: Sun Jul 8 13:45:51 2018 +0800 KYLIN-3425 Update release notes and download for 2.3.2 --- website/_docs23/release_notes.md | 21 + website/download/index.cn.md | 21 ++--- website/download/index.md| 10 +- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/website/_docs23/release_notes.md b/website/_docs23/release_notes.md index 77e59fe..fa8027a 100644 --- a/website/_docs23/release_notes.md +++ b/website/_docs23/release_notes.md @@ -15,6 +15,27 @@ or send to Apache Kylin mailing list: * User relative: [u...@kylin.apache.org](mailto:u...@kylin.apache.org) * Development relative: [d...@kylin.apache.org](mailto:d...@kylin.apache.org) +## v2.3.2 - 2018-07-08 +_Tag:_ [kylin-2.3.2](https://github.com/apache/kylin/tree/kylin-2.3.2) +This is a bug fix release after 2.3.1, with 12 bug fixes and enhancement. Check [How to upgrade](/docs23/howto/howto_upgrade.html). + +__Improvement__ +* [KYLIN-3345] - Use Apache Parent POM 19 +* [KYLIN-3372] - Upgrade jackson-databind version due to security concerns +* [KYLIN-3415] - Remove "external" module + +__Bug__ +* [KYLIN-3115] - Incompatible RowKeySplitter initialize between build and merge job +* [KYLIN-3336] - java.lang.NoSuchMethodException: org.apache.kylin.tool.HBaseUsageExtractor.execute([Ljava.lang.String;) +* [KYLIN-3348] - "missing LastBuildJobID" error when building new cube segment +* [KYLIN-3352] - Segment pruning bug, e.g. date_col > "max_date+1" +* [KYLIN-3363] - Wrong partition condition appended in JDBC Source +* [KYLIN-3388] - Data may become not correct if mappers fail during the redistribute step, "distribute by rand()" +* [KYLIN-3400] - WipeCache and createCubeDesc causes deadlock +* [KYLIN-3401] - The current using zip compress tool has an arbitrary file write vulnerability +* [KYLIN-3404] - Last optimized time detail was not showing after cube optimization + + ## v2.3.1 - 2018-03-28 _Tag:_ [kylin-2.3.1](https://github.com/apache/kylin/tree/kylin-2.3.1) This is a bug fix release after 2.3.0, with 12 bug fixes and enhancement. Check [How to upgrade](/docs23/howto/howto_upgrade.html). diff --git a/website/download/index.cn.md b/website/download/index.cn.md index 4787895..438022c 100644 --- a/website/download/index.cn.md +++ b/website/download/index.cn.md @@ -13,24 +13,23 @@ title: 下载 - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - [apache-kylin-2.4.0-bin-hbase1x.tar.gz](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.4.0/apache-kylin-2.4.0-bin-hbase1x.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.4.0/apache-kylin-2.4.0-bin-hbase1x.tar.gz.asc)\] \[[sha1](https://www.apache.org/dist/kylin/apache-kylin-2.4.0/apache-kylin-2.4.0-bin-hbase1x.tar.gz.sha1)\] - for CDH 5.7+ - [apache-kylin-2.4.0-bin-cdh57.tar.gz](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.4.0/apache-kylin-2.4.0-bin-cdh57.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.4.0/apache-kylin-2.4.0-bin-cdh57.tar.gz.asc)\] \[[sha1](https://www.apache.org/dist/kylin/apache-kylin-2.4.0/apache-kylin-2.4.0-bin-cdh57.tar.gz.tar.gz.sha1)\] - -## v2.3.1 - -- [Release notes](/docs23/release_notes.html) 和 [升级指南](/docs23/howto/howto_upgrade.html) -- 源代码下载: [apache-kylin-2.3.1-src.tar.gz](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-src.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-src.tar.gz.asc)\] \[[sha1](https://www.apache.org/dist/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-src.tar.gz.sha1)\] + v2.3.2 +- This is a bug fix release after 2.3.1, with 12 bug fixes and enhancements. For the detail list please check release notes. +- [Release notes](/docs23/release_notes.html) and [升级指南](/docs23/howto/howto_upgrade.html) +- 源代码下载: [apache-kylin-2.3.2-src.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip.asc)\] \[[sha1](https://www.apache.org/dist/kylin/apache-kylin-2.3.2/apache-kylin-2.3.2-source-release.zip.sha1)\] - 二进制包下载: - - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - [apache-kylin-2.3.1-bin-hbase1x.tar.gz](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.3.1/apache-kylin-2.3.1-hbase1x-bin.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.3.
svn commit: r27755 - /dev/kylin/apache-kylin-2.3.2-rc2/
Author: kangkaisen Date: Wed Jun 27 06:41:37 2018 New Revision: 27755 Log: Upload release artifacts to staging for Kylin 2.3.2-rc2 Added: dev/kylin/apache-kylin-2.3.2-rc2/ dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip (with props) dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc.md5 dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc.sha1 dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.md5 dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.sha1 Added: dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip == Binary file - no diff available. Propchange: dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc == --- dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc (added) +++ dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc Wed Jun 27 06:41:37 2018 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEeBJuFE1tDTLKZDIq+oiLHrw6JKYFAlszEC4ACgkQ+oiLHrw6 +JKYlyhAAt5o2FUfCLomdu2GBvMEw0xieneXCZmy/qpgaqRYumttwWbeVm5Xj01ho +BcJ6P+/qaq8fn/a3aFFQ9qlPdgBPqWc/m2dskYdggx73sL6FsEE29koJ8YeoC3fF +4NTrzqyLhqTpjoTT8Q9XY/W8d3Pox0/YJv9Xa4g+1WCrQhfenXn/gVoBXFYew7qK +FnBF9pL3m79ENGiaEtkqH2C8z+jeQnz1mVzuDNKyIJ/vrkOmXUiYrv3qDywBYbFF +PdwVqS3UUlLbiaV+k66vm4L6lQGQuI+QERUadVrhtenp2juNSGdbc+uB9Bs/OaV1 +dvYUUOcTpc4uf/Ax1TQAQyigHUe2hCtbV5owgEFrmO4y0INd0tDs9VW6SCqXsX2w +CxMgNizfj7dhnBM/RwOCVH9LRfyN51D3srbe5Nhl+0NejaaYhaVlscf+biVupgkp +qHqv+NZxzIE/yJGC5EvNq77G5OJBXZcgtfBuF+NaDF/is56CibkVkyMWkD7ICE/J +mu7zIo3BG7093p2G9IIRu6SwzZYLXPdNu+mh73rXeLhGdi+6QXvj3uNbj9aW4C+L +PxxaJomZg0qSPBDOw22V4SgMGXksF38sKGD5WgKSBviyHdlKYnZ9i6N6MS2ifRg5 +aGrdicXj8a7h4qNW3o5Exze3c6oIUjI7GB5eDwfrfzhsEGuARV4= +=ZlFe +-END PGP SIGNATURE- Added: dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc.md5 == --- dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc.md5 (added) +++ dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc.md5 Wed Jun 27 06:41:37 2018 @@ -0,0 +1 @@ +dac057b13705025c6682e86292eed4cd \ No newline at end of file Added: dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc.sha1 == --- dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc.sha1 (added) +++ dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.asc.sha1 Wed Jun 27 06:41:37 2018 @@ -0,0 +1 @@ +5d43ead13f195f2e097a85aeb2981d9c76ac074f \ No newline at end of file Added: dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.md5 == --- dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.md5 (added) +++ dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.md5 Wed Jun 27 06:41:37 2018 @@ -0,0 +1 @@ +f7b3ed50ef0273606b80fef1ee37ee9d \ No newline at end of file Added: dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.sha1 == --- dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.sha1 (added) +++ dev/kylin/apache-kylin-2.3.2-rc2/apache-kylin-2.3.2-source-release.zip.sha1 Wed Jun 27 06:41:37 2018 @@ -0,0 +1 @@ +71429dba824e514ec9b43111a2fc40bccab53cfb \ No newline at end of file
svn commit: r27754 - /dev/kylin/apache-kylin-2.3.1-rc1/
Author: kangkaisen Date: Wed Jun 27 06:34:00 2018 New Revision: 27754 Log: Recover apache-kylin-2.3.1-rc1 Added: dev/kylin/apache-kylin-2.3.1-rc1/ dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz (with props) dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc.md5 dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc.sha1 dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.md5 dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.sha1 Added: dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz == Binary file - no diff available. Propchange: dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz -- svn:mime-type = application/octet-stream Added: dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc == --- dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc (added) +++ dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc Wed Jun 27 06:34:00 2018 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v2 + +iQIcBAABCAAGBQJatn+WAAoJEMZ1EPmh75SuG3cP/3yiDYTcUoZKLqq1+F0IiagA +ZMFc6D4uGv/NiBwhdaH879xUJmtDH0oObCS7sP+fq1yDmJXBYhqB7fRUR1xFq88K +1OZKiH4zrvyK6vIn6mPYPLQN27ljG6bZq3kvGNkMQGYWTvUwy7lOfqMed/D0DeZg +Te6lSwqX1vudp9RT8j/s902WUMlUWkOB1Km+EXtWzaRLNJzXFfvtJ5JF85HyEdiQ +ycgI5w6ukEqWAurVf+sIIRB1A0tkD4iKIKUwHyHtrgv0d9LVtKuYKsgZ0p9F/qRb +X2QnTHTc3WL3JznYI+IWpLxo0ouHh9nI4J8Feo6BVN6YbwNczsCyUGK201KEzY+3 +nh2a0VrC+OHU4nCbvcFllzLx5bOvm9CRYUelkMLzvI8wX9mhcEGHMBZgjWfWdJiI +0b2IQsKbUNzOmzaoSuhxJdIBpuIMu1Ac37eYmHv8nxM/+57I4OikCQJDZeswOkJ9 +F9v7A0uqONccFw5eoLJTgtHtmp1fYImMwy2MaKQln0sbOFM3e2pPhqVg8roijaCO +r9jlskQchUWPZ9HrDMJYqT5idvBdKf9VRV6ACP7uIkuDIxHUsQK+L/RKc+6+z0V1 +cWljkMFgEEW7vvy/dI5i5FX8IHg9jX8Ovfli+Zyx7NX+FXLjYNeUbddbFm1W7ZQn +fR0nX5n5/l3Ybo+kHs1o +=g+ps +-END PGP SIGNATURE- Added: dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc.md5 == --- dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc.md5 (added) +++ dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc.md5 Wed Jun 27 06:34:00 2018 @@ -0,0 +1 @@ +ba562929956e877782064237e6010afa \ No newline at end of file Added: dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc.sha1 == --- dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc.sha1 (added) +++ dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.asc.sha1 Wed Jun 27 06:34:00 2018 @@ -0,0 +1 @@ +3441693f75ab68db2baf7172a5a256148614cfba \ No newline at end of file Added: dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.md5 == --- dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.md5 (added) +++ dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.md5 Wed Jun 27 06:34:00 2018 @@ -0,0 +1 @@ +541d8dff9e5caaae349cf61ad394b0d6 \ No newline at end of file Added: dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.sha1 == --- dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.sha1 (added) +++ dev/kylin/apache-kylin-2.3.1-rc1/apache-kylin-2.3.1-src.tar.gz.sha1 Wed Jun 27 06:34:00 2018 @@ -0,0 +1 @@ +7ff10d1770d05a8324252547f95adde990bde9dc \ No newline at end of file
svn commit: r27753 - /dev/kylin/apache-kylin-2.3.2-rc1/
Author: kangkaisen Date: Wed Jun 27 06:31:46 2018 New Revision: 27753 Log: Recover apache-kylin-2.3.1-rc1 and remove apache-kylin-2.3.2-rc1 Removed: dev/kylin/apache-kylin-2.3.2-rc1/
svn commit: r27752 - /dev/kylin/apache-kylin-2.3.1-rc1/
Author: kangkaisen Date: Wed Jun 27 06:14:34 2018 New Revision: 27752 Log: rm apache-kylin-2.3.1-rc1 Removed: dev/kylin/apache-kylin-2.3.1-rc1/
[kylin] annotated tag kylin-2.3.2 updated (8247ffc -> f9e7633)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to annotated tag kylin-2.3.2 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.3.2 was modified! *** from 8247ffc (commit) to f9e7633 (tag) tagging 8247ffc2d8a8a20381575389ffb3834d784ad4ce (commit) replaces kylin-2.3.1 by kangkaisen on Wed Jun 27 12:21:38 2018 +0800 - Log - [maven-release-plugin] copy for tag kylin-2.3.2 --- No new revisions were added by this update. Summary of changes:
[kylin] branch v2.3.2-release updated: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/v2.3.2-release by this push: new db395e6 [maven-release-plugin] prepare for next development iteration db395e6 is described below commit db395e66ae2dd122a8638ffd82416b2e0d9fbf69 Author: kangkaisen AuthorDate: Wed Jun 27 12:21:45 2018 +0800 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml | 2 +- atopcalcite/pom.xml| 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index ed844b8..ae66563 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index ee02102..bb21d28 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-common/pom.xml b/core-common/pom.xml index e8c97bf..69ee687 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 44862b7..3247877 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index eace273..5267b49 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-job/pom.xml b/core-job/pom.xml index 6acf567..51f5664 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 6e10448..7845a4b 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index c049586..a0cd237 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-storage/pom.xml b/core-storage/pom.xml index d2f8e6c..647c925 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index 4f7429e..c0b87b0 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index 788ab51..c5f4f8a 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 8eb8f67..51b955a 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index aac6b5b..e52af7d 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -29,7 +29,7 @@ kylin org.apache.kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/metrics-reporter-hive/pom.xml b/metrics-reporter-hive/pom.xml index b56b58c..f71bbd0 100644 --- a/metrics-reporter-hive/pom.xml
[kylin] branch v2.3.2-release updated: [maven-release-plugin] prepare release kylin-2.3.2
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/v2.3.2-release by this push: new 8247ffc [maven-release-plugin] prepare release kylin-2.3.2 8247ffc is described below commit 8247ffc2d8a8a20381575389ffb3834d784ad4ce Author: kangkaisen AuthorDate: Wed Jun 27 12:21:21 2018 +0800 [maven-release-plugin] prepare release kylin-2.3.2 --- assembly/pom.xml | 2 +- atopcalcite/pom.xml| 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 977ca73..ed844b8 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index cb76178..ee02102 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-common/pom.xml b/core-common/pom.xml index 8dca01f..e8c97bf 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 77996b6..44862b7 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index e6d5a10..eace273 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-job/pom.xml b/core-job/pom.xml index 1db4215..6acf567 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 35b7486..6e10448 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index bd1713f..c049586 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-storage/pom.xml b/core-storage/pom.xml index 54cc786..d2f8e6c 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index e5236ee..4f7429e 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index d11fd3c..788ab51 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/jdbc/pom.xml b/jdbc/pom.xml index ae9d335..8eb8f67 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index 9ed4efe..aac6b5b 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -29,7 +29,7 @@ kylin org.apache.kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/metrics-reporter-hive/pom.xml b/metrics-reporter-hive/pom.xml index 7f3b0e3..b56b58c 100644 --- a/metrics-reporter-hive/pom.xml +++ b/metrics-reporter
[kylin] branch v2.3.2-release updated (62d93f5 -> 593b375)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to branch v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git. discard 62d93f5 [maven-release-plugin] prepare release kylin-2.3.2 discard f7aa9d6 [maven-release-plugin] prepare for next development iteration discard 7d9c001 [maven-release-plugin] prepare release kylin-2.3.2 discard 3b61943 [maven-release-plugin] prepare for next development iteration discard 7b7ed28 [maven-release-plugin] prepare release v2.3.2-release This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/v2.3.2-release (593b375) \ O -- O -- O (62d93f5) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: assembly/pom.xml | 2 +- atopcalcite/pom.xml| 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-)
[kylin] annotated tag kylin-2.3.2 deleted (was bc52092)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to annotated tag kylin-2.3.2 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.3.2 was deleted! *** tag was bc52092 The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[kylin] annotated tag kylin-2.3.2 updated (62d93f5 -> bc52092)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to annotated tag kylin-2.3.2 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.3.2 was modified! *** from 62d93f5 (commit) to bc52092 (tag) tagging 62d93f5fa67452c7aa0ef0e7340f425c3d3c59f7 (commit) replaces kylin-2.3.1 by kangkaisen on Tue Jun 26 14:21:06 2018 +0800 - Log - [maven-release-plugin] copy for tag kylin-2.3.2 --- No new revisions were added by this update. Summary of changes:
[kylin] branch v2.3.2-release updated: [maven-release-plugin] prepare release kylin-2.3.2
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/v2.3.2-release by this push: new 62d93f5 [maven-release-plugin] prepare release kylin-2.3.2 62d93f5 is described below commit 62d93f5fa67452c7aa0ef0e7340f425c3d3c59f7 Author: kangkaisen AuthorDate: Tue Jun 26 14:20:48 2018 +0800 [maven-release-plugin] prepare release kylin-2.3.2 --- assembly/pom.xml | 2 +- atopcalcite/pom.xml| 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index ae66563..ed844b8 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index bb21d28..ee02102 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-common/pom.xml b/core-common/pom.xml index 69ee687..e8c97bf 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 3247877..44862b7 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index 5267b49..eace273 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-job/pom.xml b/core-job/pom.xml index 51f5664..6acf567 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 7845a4b..6e10448 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index a0cd237..c049586 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-storage/pom.xml b/core-storage/pom.xml index 647c925..d2f8e6c 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index c0b87b0..4f7429e 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index c5f4f8a..788ab51 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 51b955a..8eb8f67 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index e52af7d..aac6b5b 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -29,7 +29,7 @@ kylin org.apache.kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/metrics-reporter-hive/pom.xml b/metrics-reporter-hive/pom.xml index f71bbd0..b56b58c 100644 --- a/metrics-reporter-hive/pom.xml +++ b/metrics-reporter
[kylin] annotated tag kylin-2.3.2 deleted (was 9b9885d)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to annotated tag kylin-2.3.2 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.3.2 was deleted! *** tag was 9b9885d The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[kylin] branch master updated: KYLIN-3425 update Kaisen Kang public keys to KEYS
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/master by this push: new a0595c0 KYLIN-3425 update Kaisen Kang public keys to KEYS a0595c0 is described below commit a0595c07ebfcade52f10e9ac134fa3e330fc57da Author: kangkaisen AuthorDate: Mon Jun 25 15:07:18 2018 +0800 KYLIN-3425 update Kaisen Kang public keys to KEYS --- KEYS | 59 +++ 1 file changed, 59 insertions(+) diff --git a/KEYS b/KEYS index c3539ca..abee492 100644 --- a/KEYS +++ b/KEYS @@ -361,3 +361,62 @@ gufAx6TUUQp+JAjVbPom8UkOo2ypuG9w2Hrz4tcAp1DpCkv6Pv4dbHAhaPn3qs8D Ygg7DIuR/Ab6IgbzOnw= =4WOu -END PGP PUBLIC KEY BLOCK- +pub rsa4096 2018-06-20 [SC] [有效至：2020-06-19] + 78126E144D6D0D32CA64322AFA888B1EBC3A24A6 +uid [ 绝对 ] kangkaisen (apache) +sig 3FA888B1EBC3A24A6 2018-06-20 kangkaisen (apache) +sub rsa4096 2018-06-20 [E] [有效至：2020-06-19] +sig FA888B1EBC3A24A6 2018-06-20 kangkaisen (apache) + +-BEGIN PGP PUBLIC KEY BLOCK- + +mQINBFsqNbMBEADVQZZl67BJ7DnlGftQLvJrX9Ta0Ci6cJBqniX2sWULa+mpyAwl +zQJ4cwlhLBIvIReAS03ow0krNj6yNp/KfXIapSwvtpf9dLn8imAFIhkZ4PXNKamn +g5r42a30w2ItesIDNKqNATgRhRPTCJfI7D9mHR3fUn+MxeDJcpQDxJgWJjxHeEMi +p+CAX4SwiqPL0kOC82P3ofN1myBLX7WHWcmRBucvJTgVgX20d6qjjZhyNSb1uc1U +4FBDYcDtAeffH2tUkm7eYttLqOeSUrMhC13AXuEGg2Ptw2JmuWqb442osVhqZCD6 +jeeivZnFroq87a08ezU01JyoL7fe750f5Ec4Qz6lT9h3bGMSSNhISVEfwEnp53MN +LcByCrNbnnXWL+qjL7Gfa6CFS9j9ZlyL2L0RCJZWFvZgDG38aLe02EtGrDimeVkK +gexciEw3MqK1uP0T1+n3g01Lujio/hflF7HvqDO3lo6hl8ygNyenWMYys40w6WPP +RxZbqG+A0V0nkSEMY8rgdI1U/FgcAt9oMBUD4B9JA4PpNZkJg1DAGABzLYFuxiJ9 +V5y8UiotVcuV2yrWje8Bwo1j95q2m14OpnYs+G+rrbEF1ya4eFlCJ2ThwH9UiZTo +4xY5t7DjgwnCEEyxGhjoQJ9Xc9TfhaXl5dRKncnJzJFu8QiMpG4y6S/cbQARAQAB +tCtrYW5na2Fpc2VuIChhcGFjaGUpIDxrYW5na2Fpc2VuQGFwYWNoZS5vcmc+iQJU +BBMBCAA+FiEEeBJuFE1tDTLKZDIq+oiLHrw6JKYFAlsqNbMCGwMFCQPCZwAFCwkI +BwIGFQoJCAsCBBYCAwECHgECF4AACgkQ+oiLHrw6JKaMYQ/+OL7dDP2WFeUd+9ev +khFXUtPuoGrpHqmW09/TRBFAL1+yjSB/lYgjg/vP4z+Dw1E/DiIBiFEW0BOwas9/ +59fu7dWmyi/wySZPYLCkAip4s2J+qedB9a67NLV67OyD50hHoRlF9U2WV+Rx4HW+ +JurfKo1UvPtTezQMArXsfJFBQkw0PEyeV+QnQAlHRgJDNDzondYs75nYbfigzrZu +5A+tQTfa889fP+dhd/HyLndqtra3OWIxH+CIxaj7MzVn0mUdFS4hTt0eDGaJ7hXr +vazy58WvPaB9AO4it2/51+iHFVUuoXMdzsNXAFE/RNGKzLq6W+Jy4gfb1k8geo8v +fgxl7M30bmq2Jpd9yug87FCVsvBXsjuxhwUB3wkBN9nRpzWUQ/h2Pkru/iKWpXpT +YnLHWq5MPzPQJdYp1b51FxMUWDaSvmMxP54/tLzJo1bQ9lNg5LqByqMoaRtdE2jS +dZxtCmzlqmrQysWNFjttdBt6JdsZf/r880O+F3/srCcdC+LLrSOfArkDJ98Mzai8 +cE1OxsF3BtoHnrEe/iMUpxsuNr6SyYoCknChSuiL0XSh1BeLSmCXw1kRjJI8gWmF +BsTgwGIDiSaarPndnYAlow5m/cMseMxjABGdmlmLRsFTaR/nO2zGd2Yk8wZBa9tF +jdS7EbuKiSzXPK5aQQHq9bBtChy5Ag0EWyo1swEQAJ0GZKiMMbi9UEFMolp0zolt +wAUOnZEIPlqwWnoiTTAggNWo1V1NVyMsolROdnnzw7aQCh6gfK3Vh+2xyPzIvWqC +mL6r6bzFnM7jyK32+SebQUi6/sPGPmMpfanxCLI0H3VZh8MEmInHaW+JkIQ2EECi +lk8p975YVUe30X+gUH1hNjTjzlnTaI1e/F+A2ekJAn72TYXAWoT0EPjfOq+SsENa +kpqI5b3ktXyI6Mh+Bag+9s6yIjNUTzK0WaBOFXfk2t+Ieejr+qPRO5K8FPcnNWJ3 +uKbgNHLB8wk5E/orctjTvwcv6m6SQ3Kz8AGjYvJyb6vGzal54iNDID5dP5Yr/HwR +JCyLPLvzfvMaNWT1PwzFBvoDLvc+vCYPf27YESMdrk9Tm/btkeLVYEqiNvZYzoep +sqsIPM1QVOA9hNFSu/EVu0WrjXsh+2iE4pSe0sGU1XY/tJAjmljGHt3dgq27wtZQ +QP2QJugCu/Akegfh/MZLcseVZTn08YjrSVsmUEOOqe36El4e6OAmDnZvFBminw3i +NjxHZVIFXyRz+9LDeNJ81/NcIWQ7rKGeTBrjulHF5J3AhopJvKcejUvO8x4P+tYM +gDNUqB2/nzIAL04d960fsxcqae/ovhAAtcCJQWV/XAuPRXlx0Alafo4MVQ6KyfXO +9NgzLFodJ9G5j7AiyEAJABEBAAGJAjwEGAEIACYWIQR4Em4UTW0NMspkMir6iIse +vDokpgUCWyo1swIbDAUJA8JnAAAKCRD6iIsevDokptrTEACB5tKQYaBGdPCq8+y9 +44PYRjHZhMwK9a7Mr5Wa7A1DOWgTFwFMRz1IbmLwrUaJRT3bAIvd8btY/pOaZkHa +lP5QaI5k7CCV+fD0e6HMQCX9Bzra2BF4z/z1Z+bY/zfGKMGjmszHg25pQALco5ch +YFv3US/vIYy9FsYR6oLyqA8H9QihkqRd0nCUc1ZLqERMZhf3S9LrJqoCsFRpPaVP +I6fWJXvh3PH7ILPo+P1nM/ntNruJWXb50jJGgMPaAduBdW4YD+z/3ft+hnyZQBVA +cORe8A+JLtyw43fSI0lzxpdJ8Gu9GsKl3zhR9rucKLIOALb5wxWuyUHoTpb6/2sp +6Zu8Ve1u7LjQq0qx0WrV2DNg0yyLLgXGKFPcNttFtur2jafy4lqqxT92fjXqPGpg +db9Z5EtcXfIxlMpN2s6VT5kPLz19kfMOJtjRbaRJr6/+bSkgevY9iLx5x+JJQGRO +OEsej0kYZq0NzK2i0r/zC4hcCE5K2v2fhqqPXmgL1yOgaj+N05wImYal6Sjotc1W +TD/sjPXP1Mk/fRHrdm4nIFtu9T6BGhU49IfSg2PT3zEIi0tSggoOX8HYf261vWIC +GQ0EWghaKAvZFiq8+A9mTWrAQxXT+UKcFx7S9ZdDM5kE5ORJiftiuJOGIeYOjuRB +OdzEAbRO1ZpZZ82/yhRecYqxOw== +=6AF1 +-END PGP PUBLIC KEY BLOCK-
svn commit: r27703 - /dev/kylin/KEYS
Author: kangkaisen Date: Mon Jun 25 03:09:56 2018 New Revision: 27703 Log: Update KEYS for kangkaisen Modified: dev/kylin/KEYS Modified: dev/kylin/KEYS == --- dev/kylin/KEYS (original) +++ dev/kylin/KEYS Mon Jun 25 03:09:56 2018 @@ -362,3 +362,62 @@ gufAx6TUUQp+JAjVbPom8UkOo2ypuG9w2Hrz4tcA Ygg7DIuR/Ab6IgbzOnw= =4WOu -END PGP PUBLIC KEY BLOCK- +pub rsa4096 2018-06-20 [SC] [ææè³ï¼2020-06-19] + 78126E144D6D0D32CA64322AFA888B1EBC3A24A6 +uid [ ç»å¯¹ ] kangkaisen (apache) +sig 3FA888B1EBC3A24A6 2018-06-20 kangkaisen (apache) +sub rsa4096 2018-06-20 [E] [ææè³ï¼2020-06-19] +sig FA888B1EBC3A24A6 2018-06-20 kangkaisen (apache) + +-BEGIN PGP PUBLIC KEY BLOCK- + +mQINBFsqNbMBEADVQZZl67BJ7DnlGftQLvJrX9Ta0Ci6cJBqniX2sWULa+mpyAwl +zQJ4cwlhLBIvIReAS03ow0krNj6yNp/KfXIapSwvtpf9dLn8imAFIhkZ4PXNKamn +g5r42a30w2ItesIDNKqNATgRhRPTCJfI7D9mHR3fUn+MxeDJcpQDxJgWJjxHeEMi +p+CAX4SwiqPL0kOC82P3ofN1myBLX7WHWcmRBucvJTgVgX20d6qjjZhyNSb1uc1U +4FBDYcDtAeffH2tUkm7eYttLqOeSUrMhC13AXuEGg2Ptw2JmuWqb442osVhqZCD6 +jeeivZnFroq87a08ezU01JyoL7fe750f5Ec4Qz6lT9h3bGMSSNhISVEfwEnp53MN +LcByCrNbnnXWL+qjL7Gfa6CFS9j9ZlyL2L0RCJZWFvZgDG38aLe02EtGrDimeVkK +gexciEw3MqK1uP0T1+n3g01Lujio/hflF7HvqDO3lo6hl8ygNyenWMYys40w6WPP +RxZbqG+A0V0nkSEMY8rgdI1U/FgcAt9oMBUD4B9JA4PpNZkJg1DAGABzLYFuxiJ9 +V5y8UiotVcuV2yrWje8Bwo1j95q2m14OpnYs+G+rrbEF1ya4eFlCJ2ThwH9UiZTo +4xY5t7DjgwnCEEyxGhjoQJ9Xc9TfhaXl5dRKncnJzJFu8QiMpG4y6S/cbQARAQAB +tCtrYW5na2Fpc2VuIChhcGFjaGUpIDxrYW5na2Fpc2VuQGFwYWNoZS5vcmc+iQJU +BBMBCAA+FiEEeBJuFE1tDTLKZDIq+oiLHrw6JKYFAlsqNbMCGwMFCQPCZwAFCwkI +BwIGFQoJCAsCBBYCAwECHgECF4AACgkQ+oiLHrw6JKaMYQ/+OL7dDP2WFeUd+9ev +khFXUtPuoGrpHqmW09/TRBFAL1+yjSB/lYgjg/vP4z+Dw1E/DiIBiFEW0BOwas9/ +59fu7dWmyi/wySZPYLCkAip4s2J+qedB9a67NLV67OyD50hHoRlF9U2WV+Rx4HW+ +JurfKo1UvPtTezQMArXsfJFBQkw0PEyeV+QnQAlHRgJDNDzondYs75nYbfigzrZu +5A+tQTfa889fP+dhd/HyLndqtra3OWIxH+CIxaj7MzVn0mUdFS4hTt0eDGaJ7hXr +vazy58WvPaB9AO4it2/51+iHFVUuoXMdzsNXAFE/RNGKzLq6W+Jy4gfb1k8geo8v +fgxl7M30bmq2Jpd9yug87FCVsvBXsjuxhwUB3wkBN9nRpzWUQ/h2Pkru/iKWpXpT +YnLHWq5MPzPQJdYp1b51FxMUWDaSvmMxP54/tLzJo1bQ9lNg5LqByqMoaRtdE2jS +dZxtCmzlqmrQysWNFjttdBt6JdsZf/r880O+F3/srCcdC+LLrSOfArkDJ98Mzai8 +cE1OxsF3BtoHnrEe/iMUpxsuNr6SyYoCknChSuiL0XSh1BeLSmCXw1kRjJI8gWmF +BsTgwGIDiSaarPndnYAlow5m/cMseMxjABGdmlmLRsFTaR/nO2zGd2Yk8wZBa9tF +jdS7EbuKiSzXPK5aQQHq9bBtChy5Ag0EWyo1swEQAJ0GZKiMMbi9UEFMolp0zolt +wAUOnZEIPlqwWnoiTTAggNWo1V1NVyMsolROdnnzw7aQCh6gfK3Vh+2xyPzIvWqC +mL6r6bzFnM7jyK32+SebQUi6/sPGPmMpfanxCLI0H3VZh8MEmInHaW+JkIQ2EECi +lk8p975YVUe30X+gUH1hNjTjzlnTaI1e/F+A2ekJAn72TYXAWoT0EPjfOq+SsENa +kpqI5b3ktXyI6Mh+Bag+9s6yIjNUTzK0WaBOFXfk2t+Ieejr+qPRO5K8FPcnNWJ3 +uKbgNHLB8wk5E/orctjTvwcv6m6SQ3Kz8AGjYvJyb6vGzal54iNDID5dP5Yr/HwR +JCyLPLvzfvMaNWT1PwzFBvoDLvc+vCYPf27YESMdrk9Tm/btkeLVYEqiNvZYzoep +sqsIPM1QVOA9hNFSu/EVu0WrjXsh+2iE4pSe0sGU1XY/tJAjmljGHt3dgq27wtZQ +QP2QJugCu/Akegfh/MZLcseVZTn08YjrSVsmUEOOqe36El4e6OAmDnZvFBminw3i +NjxHZVIFXyRz+9LDeNJ81/NcIWQ7rKGeTBrjulHF5J3AhopJvKcejUvO8x4P+tYM +gDNUqB2/nzIAL04d960fsxcqae/ovhAAtcCJQWV/XAuPRXlx0Alafo4MVQ6KyfXO +9NgzLFodJ9G5j7AiyEAJABEBAAGJAjwEGAEIACYWIQR4Em4UTW0NMspkMir6iIse +vDokpgUCWyo1swIbDAUJA8JnAAAKCRD6iIsevDokptrTEACB5tKQYaBGdPCq8+y9 +44PYRjHZhMwK9a7Mr5Wa7A1DOWgTFwFMRz1IbmLwrUaJRT3bAIvd8btY/pOaZkHa +lP5QaI5k7CCV+fD0e6HMQCX9Bzra2BF4z/z1Z+bY/zfGKMGjmszHg25pQALco5ch +YFv3US/vIYy9FsYR6oLyqA8H9QihkqRd0nCUc1ZLqERMZhf3S9LrJqoCsFRpPaVP +I6fWJXvh3PH7ILPo+P1nM/ntNruJWXb50jJGgMPaAduBdW4YD+z/3ft+hnyZQBVA +cORe8A+JLtyw43fSI0lzxpdJ8Gu9GsKl3zhR9rucKLIOALb5wxWuyUHoTpb6/2sp +6Zu8Ve1u7LjQq0qx0WrV2DNg0yyLLgXGKFPcNttFtur2jafy4lqqxT92fjXqPGpg +db9Z5EtcXfIxlMpN2s6VT5kPLz19kfMOJtjRbaRJr6/+bSkgevY9iLx5x+JJQGRO +OEsej0kYZq0NzK2i0r/zC4hcCE5K2v2fhqqPXmgL1yOgaj+N05wImYal6Sjotc1W +TD/sjPXP1Mk/fRHrdm4nIFtu9T6BGhU49IfSg2PT3zEIi0tSggoOX8HYf261vWIC +GQ0EWghaKAvZFiq8+A9mTWrAQxXT+UKcFx7S9ZdDM5kE5ORJiftiuJOGIeYOjuRB +OdzEAbRO1ZpZZ82/yhRecYqxOw== +=6AF1 +-END PGP PUBLIC KEY BLOCK-
svn commit: r27702 - /dev/kylin/apache-kylin-2.3.2-rc1/
Author: kangkaisen Date: Mon Jun 25 02:54:13 2018 New Revision: 27702 Log: Upload release artifacts to staging Added: dev/kylin/apache-kylin-2.3.2-rc1/ dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip (with props) dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc.md5 dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc.sha1 dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.md5 dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.sha1 Added: dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip == Binary file - no diff available. Propchange: dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc == --- dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc (added) +++ dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc Mon Jun 25 02:54:13 2018 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEeBJuFE1tDTLKZDIq+oiLHrw6JKYFAlsvg18ACgkQ+oiLHrw6 +JKZIJQ//aaSx1oDvY5Z4sDsbzCS6SYxRo0If5JhB++iwMa1ktGirvcBI0tKqj4Tn +inksAVknzjZoQnxcmarcNpHhKiC8/1UyYhek2qO8NG3BA11c5QEi0d5U9XW43sRk +HBQWBxOcWzh074UmcW0QYA1AFnKrgi/6NSLEJP563bbURdLzGzNSG7U+HAwHAV/E +0Zm7PxdrALuTmIn1MibxdFeU9GcOIdDs+1uIT825Vv/+d2lUjnzWC2PCpGmkKtnI +QD9bfOqKn6PImh54iCLNF1obY6Ba8uJpj0Su5SGP+adHXm/L3tRgFF+4gKQdwtt2 +46dkQAErpt/3fsz8jPZugfbuyG8H6qu3gmvU5i2p1EWQaVSBenRYIIQmiH7PvOlY +4fkyrhI2eSVCw0lvInbvVxexwP7pcov+TsMU4TFgEq80JFy/TgMosyowp8HlUG9D +t9m6qB+gcJrSojEHS7MOwpGnEPeK744zryMiKg/3t4E02gJcuYcLUrBV0OeEiQm9 +tGFixDwyTUie+UP8OoiSnpSlwVKxfblWdsREquZBkw1xNLSyHaFB4DV+oje0gEi6 +oBbA6Rz/z+8HqG9kE/cE8T3xbtUYCnddLZ+muDoLJdB3Nnxb34I/1kTDjZ81eC0C +TasnTlT/MJ97ahqXd760ApUauYmkJ48NMaDT9Ssom66ZRNT7hKQ= +=y332 +-END PGP SIGNATURE- Added: dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc.md5 == --- dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc.md5 (added) +++ dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc.md5 Mon Jun 25 02:54:13 2018 @@ -0,0 +1 @@ +3d340bcb93d070365be3e711da05564e \ No newline at end of file Added: dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc.sha1 == --- dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc.sha1 (added) +++ dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.asc.sha1 Mon Jun 25 02:54:13 2018 @@ -0,0 +1 @@ +d11b2475ea4a875897342f5b5bbefcc668663bf9 \ No newline at end of file Added: dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.md5 == --- dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.md5 (added) +++ dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.md5 Mon Jun 25 02:54:13 2018 @@ -0,0 +1 @@ +f6babf000e247447840d323838365925 \ No newline at end of file Added: dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.sha1 == --- dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.sha1 (added) +++ dev/kylin/apache-kylin-2.3.2-rc1/apache-kylin-2.3.2-source-release.zip.sha1 Mon Jun 25 02:54:13 2018 @@ -0,0 +1 @@ +3a7681ad28ecbb6f437a4204bd92d946a5f3daa8 \ No newline at end of file
[kylin] branch v2.3.2-release updated: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/v2.3.2-release by this push: new f7aa9d6 [maven-release-plugin] prepare for next development iteration f7aa9d6 is described below commit f7aa9d6d5e9dc32fda2769bcf88fa519648c803c Author: kangkaisen AuthorDate: Sun Jun 24 19:44:22 2018 +0800 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml | 2 +- atopcalcite/pom.xml| 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index ed844b8..ae66563 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index ee02102..bb21d28 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-common/pom.xml b/core-common/pom.xml index e8c97bf..69ee687 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 44862b7..3247877 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index eace273..5267b49 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-job/pom.xml b/core-job/pom.xml index 6acf567..51f5664 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 6e10448..7845a4b 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index c049586..a0cd237 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-storage/pom.xml b/core-storage/pom.xml index d2f8e6c..647c925 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index 4f7429e..c0b87b0 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index 788ab51..c5f4f8a 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 8eb8f67..51b955a 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index aac6b5b..e52af7d 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -29,7 +29,7 @@ kylin org.apache.kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/metrics-reporter-hive/pom.xml b/metrics-reporter-hive/pom.xml index b56b58c..f71bbd0 100644 --- a/metrics-reporter-hive/pom.xml
[kylin] annotated tag kylin-2.3.2 updated (7d9c001 -> 9b9885d)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to annotated tag kylin-2.3.2 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.3.2 was modified! *** from 7d9c001 (commit) to 9b9885d (tag) tagging 7d9c001fa9332470f0ba9ac91c2fb9a4b0891651 (commit) replaces kylin-2.3.1 by kangkaisen on Sun Jun 24 19:44:16 2018 +0800 - Log - [maven-release-plugin] copy for tag kylin-2.3.2 --- No new revisions were added by this update. Summary of changes:
[kylin] branch v2.3.2-release updated: [maven-release-plugin] prepare release kylin-2.3.2
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/v2.3.2-release by this push: new 7d9c001 [maven-release-plugin] prepare release kylin-2.3.2 7d9c001 is described below commit 7d9c001fa9332470f0ba9ac91c2fb9a4b0891651 Author: kangkaisen AuthorDate: Sun Jun 24 19:43:57 2018 +0800 [maven-release-plugin] prepare release kylin-2.3.2 --- assembly/pom.xml | 2 +- atopcalcite/pom.xml| 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index ae66563..ed844b8 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index bb21d28..ee02102 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-common/pom.xml b/core-common/pom.xml index 69ee687..e8c97bf 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 3247877..44862b7 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index 5267b49..eace273 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-job/pom.xml b/core-job/pom.xml index 51f5664..6acf567 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 7845a4b..6e10448 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index a0cd237..c049586 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/core-storage/pom.xml b/core-storage/pom.xml index 647c925..d2f8e6c 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index c0b87b0..4f7429e 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index c5f4f8a..788ab51 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 51b955a..8eb8f67 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index e52af7d..aac6b5b 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -29,7 +29,7 @@ kylin org.apache.kylin -2.3.3-SNAPSHOT +2.3.2 diff --git a/metrics-reporter-hive/pom.xml b/metrics-reporter-hive/pom.xml index f71bbd0..b56b58c 100644 --- a/metrics-reporter-hive/pom.xml +++ b/metrics-reporter
[kylin] annotated tag kylin-2.3.2 deleted (was d9d0555)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to annotated tag kylin-2.3.2 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.3.2 was deleted! *** tag was d9d0555 The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[kylin] annotated tag kylin-2.3.2 updated (3b61943 -> d9d0555)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to annotated tag kylin-2.3.2 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.3.2 was modified! *** from 3b61943 (commit) to d9d0555 (tag) tagging 3b61943af27e42da8cf5fa5fc13f5332bf54eb7b (commit) replaces kylin-2.3.1 by kangkaisen on Sun Jun 24 17:26:16 2018 +0800 - Log - [maven-release-plugin] copy for tag kylin-2.3.2 --- No new revisions were added by this update. Summary of changes:
[kylin] annotated tag v2.3.2-release deleted (was 3be3461)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to annotated tag v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag v2.3.2-release was deleted! *** tag was 3be3461 The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[kylin] annotated tag v2.3.2-release updated (7b7ed28 -> 3be3461)
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a change to annotated tag v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag v2.3.2-release was modified! *** from 7b7ed28 (commit) to 3be3461 (tag) tagging 7b7ed289cc5d0766e782a17b0fea263328f2e2c3 (commit) replaces kylin-2.3.1 by kangkaisen on Sun Jun 24 16:54:43 2018 +0800 - Log - [maven-release-plugin] copy for tag v2.3.2-release --- No new revisions were added by this update. Summary of changes:
[kylin] branch v2.3.2-release updated: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/v2.3.2-release by this push: new 3b61943 [maven-release-plugin] prepare for next development iteration 3b61943 is described below commit 3b61943af27e42da8cf5fa5fc13f5332bf54eb7b Author: kangkaisen AuthorDate: Sun Jun 24 16:54:49 2018 +0800 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml | 2 +- atopcalcite/pom.xml| 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index ed844b8..ae66563 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index ee02102..bb21d28 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-common/pom.xml b/core-common/pom.xml index e8c97bf..69ee687 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 44862b7..3247877 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index eace273..5267b49 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-job/pom.xml b/core-job/pom.xml index 6acf567..51f5664 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 6e10448..7845a4b 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index c049586..a0cd237 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/core-storage/pom.xml b/core-storage/pom.xml index d2f8e6c..647c925 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index 4f7429e..c0b87b0 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index 788ab51..c5f4f8a 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 8eb8f67..51b955a 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index aac6b5b..e52af7d 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -29,7 +29,7 @@ kylin org.apache.kylin -2.3.2 +2.3.3-SNAPSHOT diff --git a/metrics-reporter-hive/pom.xml b/metrics-reporter-hive/pom.xml index b56b58c..f71bbd0 100644 --- a/metrics-reporter-hive/pom.xml
[kylin] branch v2.3.2-release updated: [maven-release-plugin] prepare release v2.3.2-release
This is an automated email from the ASF dual-hosted git repository. kangkaisen pushed a commit to branch v2.3.2-release in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/v2.3.2-release by this push: new 7b7ed28 [maven-release-plugin] prepare release v2.3.2-release 7b7ed28 is described below commit 7b7ed289cc5d0766e782a17b0fea263328f2e2c3 Author: kangkaisen AuthorDate: Sun Jun 24 16:54:26 2018 +0800 [maven-release-plugin] prepare release v2.3.2-release --- assembly/pom.xml | 2 +- atopcalcite/pom.xml| 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 977ca73..ed844b8 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index cb76178..ee02102 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-common/pom.xml b/core-common/pom.xml index 8dca01f..e8c97bf 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 77996b6..44862b7 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index e6d5a10..eace273 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-job/pom.xml b/core-job/pom.xml index 1db4215..6acf567 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 35b7486..6e10448 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index bd1713f..c049586 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/core-storage/pom.xml b/core-storage/pom.xml index 54cc786..d2f8e6c 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index e5236ee..4f7429e 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index d11fd3c..788ab51 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/jdbc/pom.xml b/jdbc/pom.xml index ae9d335..8eb8f67 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index 9ed4efe..aac6b5b 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -29,7 +29,7 @@ kylin org.apache.kylin -2.3.2-SNAPSHOT +2.3.2 diff --git a/metrics-reporter-hive/pom.xml b/metrics-reporter-hive/pom.xml index 7f3b0e3..b56b58c 100644 --- a/metrics-reporter-hive/pom.xml +++ b/metrics
kylin git commit: KYLIN-2672 Add UT
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2672 b8105db9e -> 0b3938517
KYLIN-2672 Add UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0b393851
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0b393851
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0b393851
Branch: refs/heads/KYLIN-2672
Commit: 0b393851718a17d3cae470e67e628d69cabf4ad8
Parents: b8105db
Author: kangkaisen
Authored: Fri Feb 9 15:53:02 2018 +0800
Committer: kangkaisen
Committed: Fri Feb 9 15:53:56 2018 +0800
--
.../rest/controller/CacheControllerTest.java| 54 +++-
1 file changed, 53 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0b393851/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
--
diff --git
a/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
b/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
index c2e21cc..3466a36 100644
---
a/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
+++
b/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
@@ -19,7 +19,17 @@
package org.apache.kylin.rest.controller;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.rest.request.CubeMigrationRequest;
import org.apache.kylin.rest.service.CacheService;
import org.apache.kylin.rest.service.ServiceTestBase;
import org.junit.Before;
@@ -27,6 +37,9 @@ import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
/**
* @author shaoshi
*/
@@ -48,7 +61,46 @@ public class CacheControllerTest extends ServiceTestBase {
@Test
public void testBasics() throws IOException {
-
cacheController.wipeCache("cube_desc", "drop",
"test_kylin_cube_with_slr_desc");
}
+
+@Test
+public void testClearCacheForCubeMigration() throws IOException {
+KylinConfig config = KylinConfig.getInstanceFromEnv();
+String CUBENAME = "test_kylin_cube_without_slr_desc";
+
+CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
+DataModelDesc modelDesc = cubeDesc.getModel();
+Map tableToProjects = new HashMap<>();
+for (TableRef tableRef : modelDesc.getAllTables()) {
+tableToProjects.put(tableRef.getTableIdentity(),
tableRef.getTableDesc().getProject());
+}
+
+String uuid = cubeDesc.getUuid();
+String signature = cubeDesc.getSignature();
+
+assertEquals(cubeDesc.getRetentionRange(), 0);
+
+//update cubeDesc
+cubeDesc.setRetentionRange(2018);
+cubeDesc.updateRandomUuid();
+
+//directly update metadata in store to simulate cube migration
+Serializer cubeDescSerializer = new
JsonSerializer(CubeDesc.class);
+getStore().putResource(cubeDesc.getResourcePath(), cubeDesc,
cubeDescSerializer);
+
+CubeMigrationRequest request = new CubeMigrationRequest();
+request.setCube(cubeDesc.getName());
+request.setModel(modelDesc.getName());
+request.setProject(modelDesc.getProject());
+request.setTableToProjects(tableToProjects);
+
+cacheController.clearCacheForCubeMigration(request);
+
+assertEquals(2018,
cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
+assertEquals(signature,
cubeDescManager.getCubeDesc(CUBENAME).getSignature());
+assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
+}
+
}
kylin git commit: KYLIN-2999 One click migrate cube in web
Repository: kylin
Updated Branches:
refs/heads/master 34cbd1813 -> a41a7e7d3
KYLIN-2999 One click migrate cube in web
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a41a7e7d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a41a7e7d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a41a7e7d
Branch: refs/heads/master
Commit: a41a7e7d3a97197a16360ab41e94d9ee5c77ebc6
Parents: 34cbd18
Author: kangkaisen
Authored: Tue Dec 5 20:37:08 2017 +0800
Committer: kangkaisen
Committed: Fri Jan 26 17:28:54 2018 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 25 +
.../kylin/rest/controller/CubeController.java | 7
.../apache/kylin/rest/service/CubeService.java | 39
webapp/app/js/controllers/cubes.js | 33 -
webapp/app/js/services/cubes.js | 3 +-
webapp/app/js/services/kylinProperties.js | 14 ++-
webapp/app/partials/cubes/cubes.html| 1 +
7 files changed, 117 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index cfe6fe2..4c8a3f8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1572,4 +1572,29 @@ abstract public class KylinConfigBase implements
Serializable {
return getOptional("kylin.metrics.subject-query-rpc",
"METRICS_QUERY_RPC") + "_"
+ getKylinMetricsSubjectSuffix();
}
+
+//
+// tool
+//
+public boolean isAllowAutoMigrateCube() {
+return
Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.enabled",
"false"));
+}
+
+public boolean isAutoMigrateCubeCopyAcl() {
+return
Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.copy-acl",
"true"));
+}
+
+public boolean isAutoMigrateCubePurge() {
+return
Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.purge-src-cube",
"true"));
+}
+
+public String getAutoMigrateCubeSrcConfig() {
+return getOptional("kylin.tool.auto-migrate-cube.src-config", "");
+}
+
+public String getAutoMigrateCubeDestConfig() {
+return getOptional("kylin.tool.auto-migrate-cube.dest-config", "");
+}
+
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f6c2abf..89c11a8 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -1022,6 +1022,13 @@ public class CubeController extends BasicController {
}
}
+@RequestMapping(value = "/{cube}/{project}/migrate", method = {
RequestMethod.POST })
+@ResponseBody
+public void migrateCube(@PathVariable String cube, @PathVariable String
project) {
+CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cube);
+cubeService.migrateCube(cubeInstance, project);
+}
+
public void setCubeService(CubeService cubeService) {
this.cubeService = cubeService;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a81d189..154354d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -30,6 +30,7 @@ import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common
kylin git commit: KYLIN-3117 Hide project configs in cube level
Repository: kylin
Updated Branches:
refs/heads/master 018605d64 -> 1d1627a9a
KYLIN-3117 Hide project configs in cube level
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d1627a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d1627a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d1627a9
Branch: refs/heads/master
Commit: 1d1627a9a49b31c0ff96f6c9bc313587e52601c7
Parents: 018605d
Author: kangkaisen
Authored: Tue Dec 19 12:26:47 2017 +0800
Committer: kangkaisen
Committed: Fri Dec 22 18:43:00 2017 +0800
--
.../src/main/java/org/apache/kylin/cube/model/CubeDesc.java| 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d1627a9/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5ff52c1..c4e1ced 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -596,16 +596,18 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
List ownerPrj =
ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name);
// cube inherit the project override props
+LinkedHashMap allOverrideProps =
Maps.newLinkedHashMap(overrideKylinProps);
+
if (ownerPrj.size() == 1) {
Map prjOverrideProps =
ownerPrj.get(0).getOverrideKylinProps();
for (Entry entry : prjOverrideProps.entrySet()) {
if (!overrideKylinProps.containsKey(entry.getKey())) {
-overrideKylinProps.put(entry.getKey(), entry.getValue());
+allOverrideProps.put(entry.getKey(), entry.getValue());
}
}
}
-this.config = KylinConfigExt.createInstance(config,
overrideKylinProps);
+this.config = KylinConfigExt.createInstance(config, allOverrideProps);
checkArgument(this.rowkey.getRowKeyColumns().length <=
this.config.getCubeRowkeyMaxSize(),
"Too many rowkeys (%s) in CubeDesc, please try to reduce
dimension number or adopt derived dimensions",
kylin git commit: KYLIN-3091 Add a test case for IntersectDistinct
Repository: kylin Updated Branches: refs/heads/master 60431f464 -> f0e5e376d KYLIN-3091 Add a test case for IntersectDistinct Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0e5e376 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0e5e376 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0e5e376 Branch: refs/heads/master Commit: f0e5e376d6466891873514f76c7b34c73c0ea28f Parents: 60431f4 Author: kangkaisen Authored: Fri Dec 15 16:44:45 2017 +0800 Committer: kangkaisen Committed: Fri Dec 15 20:29:04 2017 +0800 -- .../query/sql_intersect_count/query03.sql | 23 1 file changed, 23 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/f0e5e376/kylin-it/src/test/resources/query/sql_intersect_count/query03.sql -- diff --git a/kylin-it/src/test/resources/query/sql_intersect_count/query03.sql b/kylin-it/src/test/resources/query/sql_intersect_count/query03.sql new file mode 100644 index 000..3cd10b1 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_intersect_count/query03.sql @@ -0,0 +1,23 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +select LEAF_CATEG_ID, +intersect_count(TEST_COUNT_DISTINCT_BITMAP, CAL_DT, array[date'2012-01-01']) as first_day +from test_kylin_fact +where CAL_DT in (date'2012-01-01',date'2012-01-02',date'2012-01-03') +group by LEAF_CATEG_ID +
kylin git commit: KYLIN-3087 Add try catch in GlobalDictionaryBuilder to release lock
Repository: kylin
Updated Branches:
refs/heads/master 280f6738b -> 60431f464
KYLIN-3087 Add try catch in GlobalDictionaryBuilder to release lock
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/60431f46
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/60431f46
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/60431f46
Branch: refs/heads/master
Commit: 60431f46494aaa1297d8da87bbf49bc78312fcb4
Parents: 280f673
Author: dengfangyuan
Authored: Fri Dec 15 13:28:55 2017 +0800
Committer: kangkaisen
Committed: Fri Dec 15 15:29:25 2017 +0800
--
.../java/org/apache/kylin/dict/GlobalDictionaryBuilder.java | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/60431f46/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
index 8250fed..5717542 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
@@ -53,7 +53,13 @@ public class GlobalDictionaryBuilder implements
IDictionaryBuilder {
hdfsDir =
KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
}
String baseDir = hdfsDir + "resources/GlobalDict" +
dictInfo.getResourceDir() + "/";
-this.builder = new AppendTrieDictionaryBuilder(baseDir,
maxEntriesPerSlice, true);
+
+try {
+this.builder = new AppendTrieDictionaryBuilder(baseDir,
maxEntriesPerSlice, true);
+} catch (Throwable e) {
+lock.unlock(getLockPath(sourceColumn));
+throw new RuntimeException(String.format("Failed to create global
dictionary on %s ", sourceColumn), e);
+}
this.baseId = baseId;
}
kylin git commit: KYLIN-3055 Fix NPE for intersect_count query
Repository: kylin
Updated Branches:
refs/heads/master 14dd92e00 -> 9265e150d
KYLIN-3055 Fix NPE for intersect_count query
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9265e150
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9265e150
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9265e150
Branch: refs/heads/master
Commit: 9265e150d80519d3e4f532c5f106e6718543daba
Parents: 14dd92e
Author: yuchuqian
Authored: Mon Dec 4 14:30:28 2017 +0800
Committer: kangkaisen
Committed: Wed Dec 6 20:09:44 2017 +0800
--
.../gtrecord/GTCubeStorageQueryBase.java| 11 ++--
.../query/sql_intersect_count/query01.sql | 27
2 files changed, 36 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9265e150/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index e9c6885..024990f 100644
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -39,6 +39,7 @@ import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.gridtable.StorageLimitLevel;
import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.bitmap.BitmapMeasureType;
import org.apache.kylin.metadata.filter.CaseTupleFilter;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -143,7 +144,7 @@ public abstract class GTCubeStorageQueryBase implements
IStorageQuery {
// exactAggregation mean: needn't aggregation at storage and query
engine both.
boolean exactAggregation = isExactAggregation(context, cuboid, groups,
otherDimsD, singleValuesD,
-derivedPostAggregation, sqlDigest.aggregations);
+derivedPostAggregation, sqlDigest.aggregations,
sqlDigest.aggrSqlCalls);
context.setExactAggregation(exactAggregation);
// replace derived columns in filter with host columns; columns on
loosened condition must be added to group by
@@ -518,7 +519,7 @@ public abstract class GTCubeStorageQueryBase implements
IStorageQuery {
private boolean isExactAggregation(StorageContext context, Cuboid cuboid,
Collection groups,
Set othersD, Set singleValuesD,
Set derivedPostAggregation,
-Collection functionDescs) {
+Collection functionDescs, List
aggrSQLCalls) {
if (context.isNeedStorageAggregation()) {
logger.info("exactAggregation is false because need storage
aggregation");
return false;
@@ -550,6 +551,12 @@ public abstract class GTCubeStorageQueryBase implements
IStorageQuery {
return false;
}
}
+for (SQLDigest.SQLCall aggrSQLCall : aggrSQLCalls) {
+if
(aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT)) {
+logger.info("exactAggregation is false because has
INTERSECT_COUNT");
+return false;
+}
+}
// for partitioned cube, the partition column must belong to group by
or has single value
PartitionDesc partDesc =
cuboid.getCubeDesc().getModel().getPartitionDesc();
http://git-wip-us.apache.org/repos/asf/kylin/blob/9265e150/kylin-it/src/test/resources/query/sql_intersect_count/query01.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_intersect_count/query01.sql
b/kylin-it/src/test/resources/query/sql_intersect_count/query01.sql
new file mode 100644
index 000..7fd5ca2
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_intersect_count/query01.sql
@@ -0,0 +1,27 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an &q
kylin git commit: KYLIN-2604 Non-Int type precise count distinct measure must set advanced dict
Repository: kylin
Updated Branches:
refs/heads/master 5a6e6d7e3 -> ed319575e
KYLIN-2604 Non-Int type precise count distinct measure must set advanced dict
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ed319575
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ed319575
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ed319575
Branch: refs/heads/master
Commit: ed319575e6da35998e427e9803f69df2b450b872
Parents: 5a6e6d7
Author: kangkaisen
Authored: Thu Aug 24 17:49:50 2017 +0800
Committer: kangkaisen
Committed: Tue Dec 5 10:35:24 2017 +0800
--
webapp/app/js/controllers/cubeEdit.js | 12 +++-
webapp/app/js/controllers/cubeSchema.js | 20
2 files changed, 31 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed319575/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js
b/webapp/app/js/controllers/cubeEdit.js
index 99b298c..3cf248f 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -200,6 +200,16 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q,
$routeParams, $locatio
return type;
};
+ $scope.isIntMeasure = function (measure) {
+var column = measure.function.parameter.value;
+if(column && (typeof column=="string")){
+ var colType = $scope.getColumnType(VdmUtil.removeNameSpace(column),
VdmUtil.getNameSpaceAliasName(column));
+
if(colType==="tinyint"||colType==="smallint"||colType==="int"||colType==="integer"){
+return true;
+ }
+}
+return false;
+ };
// ~ Define data
$scope.state = {
@@ -730,7 +740,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q,
$routeParams, $locatio
var distinctMeasures = [];
angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
- if (measure.function.expression === 'COUNT_DISTINCT' &&
measure.function.returntype === 'bitmap') {
+ if (measure.function.expression === 'COUNT_DISTINCT' &&
measure.function.returntype === 'bitmap' && !$scope.isIntMeasure(measure)) {
var measureColumn = measure.function.parameter.value;
distinctMeasures.push(measureColumn);
//keep backward compatibility
http://git-wip-us.apache.org/repos/asf/kylin/blob/ed319575/webapp/app/js/controllers/cubeSchema.js
--
diff --git a/webapp/app/js/controllers/cubeSchema.js
b/webapp/app/js/controllers/cubeSchema.js
index a912c72..590168b 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -401,6 +401,26 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope,
QueryService, UserServic
}
+angular.forEach($scope.cubeMetaFrame.measures, function (measure,
index) {
+if (measure.function.expression === 'COUNT_DISTINCT' &&
measure.function.returntype === 'bitmap' && !$scope.isIntMeasure(measure)) {
+var measureColumn = measure.function.parameter.value;
+
+var isColumnExit = false;
+angular.forEach($scope.cubeMetaFrame.dictionaries, function
(dictionaries) {
+if (!isColumnExit) {
+//keep backward compatibility
+if (dictionaries.column == measureColumn ||
dictionaries.column == VdmUtil.removeNameSpace(measureColumn))
+isColumnExit = true;
+}
+});
+
+if (!isColumnExit) {
+errors.push("The non-Int type precise count distinct
measure must set advanced cict: " + measureColumn);
+}
+}
+});
+
+
var errorInfo = "";
angular.forEach(errors,function(item){
errorInfo+="\n"+item;
kylin git commit: KYLIN-2992 Avoid GC in CubeHFileJob.Reducer
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2992 [created] b837071a6
KYLIN-2992 Avoid GC in CubeHFileJob.Reducer
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b837071a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b837071a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b837071a
Branch: refs/heads/KYLIN-2992
Commit: b837071a6048433a0ec1708f358a62a8e90c2d1a
Parents: 31b16c3
Author: kangkaisen
Authored: Fri Sep 22 13:36:23 2017 +0800
Committer: kangkaisen
Committed: Fri Nov 3 17:33:52 2017 +0800
--
.../storage/hbase/steps/CreateHTableJob.java| 8 +-
.../kylin/storage/hbase/steps/CubeHFileJob.java | 26 --
.../storage/hbase/steps/CubeHFileMapper.java| 14 ++--
.../storage/hbase/steps/RowKeyWritable.java | 86
.../hbase/steps/CubeHFileMapper2Test.java | 4 -
.../hbase/steps/CubeHFileMapperTest.java| 8 +-
6 files changed, 123 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b837071a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index b2bab47..c41df06 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -29,7 +29,7 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
@@ -323,6 +323,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
double accumulatedSize = 0;
int j = 0;
for (Long cuboid : allCuboids) {
+
if (accumulatedSize >= hfileSizeMB) {
logger.info(String.format("Region %d's hfile %d size is
%.2f mb", i, j, accumulatedSize));
byte[] split = new
byte[RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN];
@@ -337,10 +338,11 @@ public class CreateHTableJob extends AbstractHadoopJob {
}
-SequenceFile.Writer hfilePartitionWriter =
SequenceFile.createWriter(hbaseConf,
SequenceFile.Writer.file(hfilePartitionFile),
SequenceFile.Writer.keyClass(ImmutableBytesWritable.class),
SequenceFile.Writer.valueClass(NullWritable.class));
+SequenceFile.Writer hfilePartitionWriter =
SequenceFile.createWriter(hbaseConf,
SequenceFile.Writer.file(hfilePartitionFile),
SequenceFile.Writer.keyClass(RowKeyWritable.class),
SequenceFile.Writer.valueClass(NullWritable.class));
for (int i = 0; i < splits.size(); i++) {
-hfilePartitionWriter.append(new
ImmutableBytesWritable(splits.get(i)), NullWritable.get());
+//when we compare the rowkey, we compare the row firstly.
+hfilePartitionWriter.append(new
RowKeyWritable(KeyValue.createFirstOnRow(splits.get(i)).createKeyOnly(false).getKey()),
NullWritable.get());
}
hfilePartitionWriter.close();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b837071a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 5f51c13..093e8ee 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -25,8 +25,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
@@ -39,6 +40,7 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.
kylin git commit: KYLIN-2764 Apply KYLIN-2800
Repository: kylin
Updated Branches:
refs/heads/2764 3941b4da3 -> 48f3fb195
KYLIN-2764 Apply KYLIN-2800
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/48f3fb19
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/48f3fb19
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/48f3fb19
Branch: refs/heads/2764
Commit: 48f3fb1953a413acfdd405539a7cfd211a5e85de
Parents: 3941b4d
Author: kangkaisen
Authored: Sun Oct 22 13:35:15 2017 +0800
Committer: kangkaisen
Committed: Sun Oct 22 13:47:25 2017 +0800
--
.../java/org/apache/kylin/cube/CubeManager.java | 30 +--
.../apache/kylin/dict/DictionaryManager.java| 40
2 files changed, 1 insertion(+), 69 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/48f3fb19/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 180c3f4..b18d40f 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -833,26 +833,6 @@ public class CubeManager implements IRealizationProvider {
//
-
-/**
- * Get the columns which need build the dictionary from fact table. (the
column exists on fact and is not fk)
- * @param cubeDesc
- * @return
- * @throws IOException
- */
-public List getAllDictColumnsOnFact(CubeDesc cubeDesc) throws
IOException {
-List factDictCols = new ArrayList();
-DictionaryManager dictMgr = DictionaryManager.getInstance(config);
-for (TblColRef col : cubeDesc.getAllColumnsNeedDictionaryBuilt()) {
-
-String scanTable = dictMgr.decideSourceData(cubeDesc.getModel(),
col).getTable();
-if (cubeDesc.getModel().isFactTable(scanTable)) {
-factDictCols.add(col);
-}
-}
-return factDictCols;
-}
-
public List getAllGlobalDictColumns(CubeDesc cubeDesc) {
List globalDictCols = new ArrayList();
List dictionaryDescList = cubeDesc.getDictionaries();
@@ -874,19 +854,11 @@ public class CubeManager implements IRealizationProvider {
List uhcColumns = new ArrayList();
uhcColumns.addAll(getAllGlobalDictColumns(cubeDesc));
uhcColumns.addAll(cubeDesc.getShardByColumns());
-
-//handle PK-FK, see getAllDictColumnsOnFact
-try {
-uhcColumns.retainAll(getAllDictColumnsOnFact(cubeDesc));
-} catch (IOException e) {
-throw new RuntimeException("Get all dict columns on fact failed");
-}
-
return uhcColumns;
}
public int[] getUHCIndex(CubeDesc cubeDesc) throws IOException {
-List factDictCols = getAllDictColumnsOnFact(cubeDesc);
+List factDictCols =
Lists.newArrayList(cubeDesc.getAllColumnsNeedDictionaryBuilt());
List uhcColumns = getAllUHCColumns(cubeDesc);
int[] uhcIndex = new int[factDictCols.size()];
http://git-wip-us.apache.org/repos/asf/kylin/blob/48f3fb19/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 7ce608b..e97899c 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -29,17 +29,13 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.DataModelManager;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.IReadableTable;
import org.apache.kylin.source.IReadableTable.TableSignature;
@@ -346,42 +342,6 @@ public class DictionaryManager {
return dictInfo;
}
-/**
- * Decide a dictionary&
[33/50] [abbrv] kylin git commit: Revert "#949 Refine column family for measures"
Revert "#949 Refine column family for measures"
This reverts commit 906b8f40b381ab9086a4f7e7593ae7d8391189bd.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32d346b4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32d346b4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32d346b4
Branch: refs/heads/2622-2764
Commit: 32d346b4cd619a38461a27115fbef2bd44f28372
Parents: b50030d
Author: Hongbin Ma
Authored: Thu Aug 31 22:41:20 2017 +0800
Committer: Hongbin Ma
Committed: Thu Aug 31 22:41:20 2017 +0800
--
.../org/apache/kylin/gridtable/GTRecord.java | 19 +--
1 file changed, 13 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/32d346b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 00e355f..d8c0fbd 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -45,7 +45,7 @@ public class GTRecord implements Comparable,
Cloneable {
}
this.info = info;
}
-
+
@Override
public GTRecord clone() { // deep copy
ByteArray[] cols = new ByteArray[this.cols.length];
@@ -114,6 +114,7 @@ public class GTRecord implements Comparable,
Cloneable {
return result;
}
+
/** decode and return the values of this record */
public Object[] getValues(int[] selectedColumns, Object[] result) {
assert selectedColumns.length <= result.length;
@@ -294,21 +295,27 @@ public class GTRecord implements Comparable,
Cloneable {
buf.position(pos);
}
}
-
-public void loadColumnsFromColumnBlocks(ImmutableBitSet[]
selectedColumnBlocks, ImmutableBitSet selectedCols,
-ByteBuffer buf) {
+
+public void loadColumnsFromColumnBlocks(ImmutableBitSet[]
selectedColumnBlocks, ImmutableBitSet selectedCols, ByteBuffer buf) {
+
int pos = buf.position();
+
for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) {
+
for (int i = 0; i < selectedColBlock.trueBitCount(); i++) {
+
int c = selectedColBlock.trueBitAt(i);
+
int len = info.codeSystem.codeLength(c, buf);
-if (selectedCols.get(c)) {
-cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
+if(selectedCols.get(c)) {
+cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
}
pos += len;
buf.position(pos);
}
+
}
+
}
/** change pointers to point to data in given buffer, this
[29/50] [abbrv] kylin git commit: KYLIN-2790 Add extending point for column family
KYLIN-2790 Add extending point for column family
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d62cb58b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d62cb58b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d62cb58b
Branch: refs/heads/2622-2764
Commit: d62cb58bde55766dbe87bbad29953dd1243c2014
Parents: 4782289
Author: auphyroc99 <454530...@qq.com>
Authored: Wed Aug 16 10:34:02 2017 +0800
Committer: Roger Shi
Committed: Wed Aug 30 18:13:52 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4
.../org/apache/kylin/cube/model/CubeDesc.java | 18 ++--
.../kylin/cube/model/HBaseMappingDesc.java | 18
.../org/apache/kylin/gridtable/GTRecord.java| 22
4 files changed, 60 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a113327..47806a5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -286,6 +286,10 @@ abstract public class KylinConfigBase implements
Serializable {
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}
+
+public String getHBaseMappingAdapter() {
+return getOptional("kylin.metadata.hbasemapping-adapter");
+}
//
// DICTIONARY & SNAPSHOT
http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 0e22587..d263e1f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -22,6 +22,7 @@ import static
com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
+import java.lang.reflect.Method;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
@@ -603,8 +604,21 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
validateAggregationGroups(); // check if aggregation group is valid
validateAggregationGroupsCombination();
-if (hbaseMapping != null) {
-hbaseMapping.init(this);
+String hbaseMappingAdapterName = config.getHBaseMappingAdapter();
+
+if (hbaseMappingAdapterName != null) {
+try {
+Class hbaseMappingAdapterClass =
Class.forName(hbaseMappingAdapterName);
+Method initMethod =
hbaseMappingAdapterClass.getMethod("initHBaseMapping", CubeDesc.class);
+initMethod.invoke(null, this);
+} catch (Exception e) {
+logger.error("Wrong configuration for
kylin.metadata.hbasemapping-adapter: class "
++ hbaseMappingAdapterName + " not found. ");
+}
+} else {
+if (hbaseMapping != null) {
+hbaseMapping.init(this);
+}
}
initMeasureReferenceToColumnFamily();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index d1e5829..77b4b14 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -87,6 +87,24 @@ public class HBaseMappingDesc implements
java.io.Serializable {
}
}
+public void initAsSeparatedColumns(CubeDesc cubeDesc) {
+cubeRef = cubeDesc;
+
+int cfNum = cubeDesc.getMeasures().size();
+columnFamily = new HBaseColumnFamilyDesc[cfNum];
+
+for (int i = 0; i < cfNum; i++) {
+HBaseColumnFamilyDesc cf = new
[48/50] [abbrv] kylin git commit: KYLIN-2819 add configurable zookeeper base path
KYLIN-2819 add configurable zookeeper base path
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/520f6279
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/520f6279
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/520f6279
Branch: refs/heads/2622-2764
Commit: 520f6279c560f457c1b7fe1173532959fcee5c6c
Parents: 14058bd
Author: shenyinjie
Authored: Sun Sep 3 17:36:02 2017 +0800
Committer: Li Yang
Committed: Sun Sep 3 20:46:58 2017 +0800
--
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4
core-common/src/main/resources/kylin-defaults.properties | 2 ++
.../apache/kylin/job/impl/threadpool/DistributedScheduler.java | 2 +-
.../kylin/storage/hbase/util/ZookeeperDistributedLock.java | 2 +-
4 files changed, 8 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/520f6279/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 9533400..749b515 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -226,6 +226,10 @@ abstract public class KylinConfigBase implements
Serializable {
return cachedHdfsWorkingDirectory;
}
+public String getZookeeperBasePath() {
+return getOptional("kylin.env.zookeeper-base-path", "/kylin");
+}
+
/**
* A comma separated list of host:port pairs, each corresponding to a
ZooKeeper server
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/520f6279/core-common/src/main/resources/kylin-defaults.properties
--
diff --git a/core-common/src/main/resources/kylin-defaults.properties
b/core-common/src/main/resources/kylin-defaults.properties
index f0328fa..a3277ce 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -26,6 +26,8 @@ kylin.env.hdfs-working-dir=/kylin
# DEV|QA|PROD. DEV will turn on some dev features, QA and PROD has no
difference in terms of functions.
kylin.env=QA
+# kylin zk base path
+kylin.env.zookeeper-base-path=/kylin
### SERVER | WEB ###
http://git-wip-us.apache.org/repos/asf/kylin/blob/520f6279/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index e3a5836..6d41c5e 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -82,7 +82,7 @@ public class DistributedScheduler implements
Scheduler, Conn
private String serverName;
private final static String SEGMENT_ID = "segmentId";
-public static final String ZOOKEEPER_LOCK_PATH = "/job_engine/lock"; //
note ZookeeperDistributedLock will ensure zk path prefix: /kylin/metadata
+public static final String ZOOKEEPER_LOCK_PATH = "/job_engine/lock"; //
note ZookeeperDistributedLock will ensure zk path prefix:
/${kylin.env.zookeeper-base-path}/metadata
//only for it test
public static DistributedScheduler getInstance(KylinConfig config) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/520f6279/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
index 63ffda0..844d9ed 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
@@ -102,7 +102,7 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
public Factory(KylinConfig config) {
this.curator = getZKClient(config);
-this.zkPathBase = fixSlash("/kylin/" +
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix());
+this.zkPathBase = fixSlash(config.getZookeeperBasePath() + "/" +
config.getMetadataUrlPrefix());
[42/50] [abbrv] kylin git commit: Merge commit '1004a853225503e00f48a27949d7e09109e937db'
Merge commit '1004a853225503e00f48a27949d7e09109e937db'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/34596d35
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/34596d35
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/34596d35
Branch: refs/heads/2622-2764
Commit: 34596d3553a7694738f48b10d38384c12cb535a3
Parents: 75fbdcf 1004a85
Author: Li Yang
Authored: Sun Sep 3 09:10:43 2017 +0800
Committer: Li Yang
Committed: Sun Sep 3 09:22:34 2017 +0800
--
.../org/apache/kylin/cube/model/CubeDesc.java | 1 -
.../kylin/job/execution/ExecutableManager.java | 11 +
.../engine/mr/common/AbstractHadoopJob.java | 4 +++
.../java/org/apache/kylin/jdbc/KylinClient.java | 2 ++
.../resources/query/sql_derived/query13.sql | 26
5 files changed, 12 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/34596d35/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --cc core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index be523ed,0e22587..3cbba50
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@@ -189,7 -189,7 +189,6 @@@ public class CubeDesc extends RootPersi
@JsonInclude(JsonInclude.Include.NON_NULL)
private int parentForward = 3;
- // lazy init
-// cuboid scheduler lazy built
transient private CuboidScheduler cuboidScheduler = null;
public boolean isEnableSharding() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/34596d35/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --cc
engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 9950c69,54f77c0..081ac93
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@@ -501,12 -501,12 +501,16 @@@ public abstract class AbstractHadoopJo
// write kylin.properties
File kylinPropsFile = new File(metaDir, "kylin.properties");
-kylinConfig.writeProperties(kylinPropsFile);
+kylinConfig.exportToFile(kylinPropsFile);
+
+if (prj != null) {
+
dumpList.add(ProjectManager.getInstance(kylinConfig).getProject(prj).getResourcePath());
+}
+ if (prj != null) {
+
dumpList.add(ProjectManager.getInstance(kylinConfig).getProject(prj).getResourcePath());
+ }
+
// write resources
JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir, dumpList);
http://git-wip-us.apache.org/repos/asf/kylin/blob/34596d35/kylin-it/src/test/resources/query/sql_derived/query13.sql
--
diff --cc kylin-it/src/test/resources/query/sql_derived/query13.sql
index 754495f,754495f..000
deleted file mode 100755,100755
--- a/kylin-it/src/test/resources/query/sql_derived/query13.sql
+++ /dev/null
@@@ -1,26 -1,26 +1,0 @@@
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--
--select test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name,
test_category_groupings.meta_categ_name, sum(test_kylin_fact.price) as gmv,
count(*) as trans_cnt
-- from test_kylin_fact
-- inner JOIN edw.test_cal_dt as test_cal_dt
-- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
-- inner JOIN test_category_groupings
-- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
-- where test_cal_dt.week_beg_dt is not null
-- group by test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name,
test_category_groupings.meta_categ_name
[05/50] [abbrv] kylin git commit: KYLIN-2766 use Kylin working dir to get FileSystem
KYLIN-2766 use Kylin working dir to get FileSystem
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4316cfdb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4316cfdb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4316cfdb
Branch: refs/heads/2622-2764
Commit: 4316cfdbe21cbab5764c2bcb6a45010bef434bf3
Parents: 891caab
Author: shaofengshi
Authored: Sun Aug 20 21:25:17 2017 +0800
Committer: shaofengshi
Committed: Tue Aug 22 20:52:45 2017 +0800
--
.../org/apache/kylin/engine/mr/common/HadoopCmdOutput.java| 4 ++--
.../hive/cardinality/HiveColumnCardinalityUpdateJob.java | 3 ++-
.../java/org/apache/kylin/storage/hbase/HBaseConnection.java | 2 +-
.../org/apache/kylin/storage/hbase/HBaseResourceStore.java| 7 ---
.../org/apache/kylin/storage/hbase/steps/CreateHTableJob.java | 3 ++-
.../storage/hbase/steps/HDFSPathGarbageCollectionStep.java| 2 +-
.../apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java | 5 +++--
7 files changed, 15 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/4316cfdb/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index 11d6d2c..0bff511 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -22,11 +22,11 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.FileSystemCounter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskCounter;
+import org.apache.kylin.common.util.HadoopUtil;
import
org.apache.kylin.engine.mr.steps.FactDistinctColumnsMapper.RawDataCounter;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.slf4j.Logger;
@@ -99,7 +99,7 @@ public class HadoopCmdOutput {
mapInputRecords =
String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
rawInputBytesRead =
String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue());
-String fsScheme =
FileSystem.get(job.getConfiguration()).getScheme();
+String fsScheme = HadoopUtil.getWorkingFileSystem().getScheme();
long bytesWritten = counters.findCounter(fsScheme,
FileSystemCounter.BYTES_WRITTEN).getValue();
if (bytesWritten == 0) {
bytesWritten = counters.findCounter("FileSystemCounters",
"HDFS_BYTES_WRITTEN").getValue();
http://git-wip-us.apache.org/repos/asf/kylin/blob/4316cfdb/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index 246822c..24abe77 100644
---
a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++
b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TableExtDesc;
@@ -125,7 +126,7 @@ public class HiveColumnCardinalityUpdateJob extends
AbstractHadoopJob {
}
private static List readLines(Path location, Configuration conf)
throws Exception {
-FileSystem fileSystem = FileSystem.get(location.toUri(), conf);
+FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
CompressionCodecFactory factory = new CompressionCodecFactory(conf);
FileStatus[] items = fileSystem.listStatus(location);
if (items == null)
http://git-wip-us.apache.org/repos/asf/kylin/blob/4316cfdb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnecti
[40/50] [abbrv] kylin git commit: KYLIN-2818 Refactor dateRange & sourceOffset on CubeSegment
http://git-wip-us.apache.org/repos/asf/kylin/blob/75fbdcff/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentRange.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentRange.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentRange.java
new file mode 100644
index 000..e52da09
--- /dev/null
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentRange.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * SegmentRange and TSRange seem similar but are different concepts.
+ *
+ * - SegmentRange defines the range of a segment.
+ * - TSRange is the time series range of the segment data.
+ * - When segment range is defined by time, the two can be the same, in that
case TSRange is a kind of SegmentRange.
+ * - Duration segment creation (build/refresh/merge), a new segment is defined
by either one of the two, not both.
+ * - And the choice must be consistent across all following segment creation.
+ */
+@SuppressWarnings("serial")
+public class SegmentRange implements Serializable {
+
+public final Endpoint start;
+public final Endpoint end;
+
+public SegmentRange(Endpoint start, Endpoint end) {
+this.start = start;
+this.end = end;
+checkState();
+}
+
+public SegmentRange(T start, T end) {
+if (start != null && end != null && start.getClass() != end.getClass())
+throw new IllegalArgumentException();
+
+this.start = new Endpoint(start, start == null, false);
+this.end = new Endpoint(end, false, end == null);
+checkState();
+}
+
+private void checkState() {
+if (start.compareTo(end) > 0)
+throw new IllegalStateException();
+}
+
+public boolean isInfinite() {
+return start.isMin && end.isMax;
+}
+
+public boolean contains(SegmentRange o) {
+return this.start.compareTo(o.start) <= 0 && o.end.compareTo(this.end)
<= 0;
+}
+
+public boolean overlaps(SegmentRange o) {
+return this.start.compareTo(o.end) < 0 && o.start.compareTo(this.end)
< 0;
+}
+
+public boolean connects(SegmentRange o) {
+return this.end.compareTo(o.start) == 0;
+}
+
+public boolean apartBefore(SegmentRange o) {
+return this.end.compareTo(o.start) < 0;
+}
+
+@Override
+public String toString() {
+return this.getClass().getSimpleName() + "[" + start + "," + end + ")";
+}
+
+@Override
+public int hashCode() {
+final int prime = 31;
+int result = 1;
+result = prime * result + ((end == null) ? 0 : end.hashCode());
+result = prime * result + ((start == null) ? 0 : start.hashCode());
+return result;
+}
+
+@Override
+public boolean equals(Object obj) {
+if (this == obj)
+return true;
+if (obj == null)
+return false;
+if (getClass() != obj.getClass())
+return false;
+SegmentRange other = (SegmentRange) obj;
+if (end == null) {
+if (other.end != null)
+return false;
+} else if (!end.equals(other.end))
+return false;
+if (start == null) {
+if (other.start != null)
+return false;
+} else if (!start.equals(other.start))
+return false;
+return true;
+}
+
+//
+
+public static class TSRange extends SegmentRange {
+
+public TSRange(Long start, Long end) {
+// [0, Long.MAX_VALUE) is full build (for historic reason)
+super(new Endpoint(isInfinite(start, end) ? 0 : start,
isInfinite(start, end), false), //
+new Endpoint(isInfinite(start, end) ? Long.MAX_VALUE :
end, false, isInfinite(start, end)));
+}
+
+private static boolean isInfinite(Long start, Long end) {
+return (start == null |
[26/50] [abbrv] kylin git commit: KYLIN-2773 Test for join casting
KYLIN-2773 Test for join casting
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1eb3617c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1eb3617c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1eb3617c
Branch: refs/heads/2622-2764
Commit: 1eb3617cf1cf50da16d5f86b4f922b3e8925de4b
Parents: 87f1732
Author: Roger Shi
Authored: Tue Aug 29 19:22:18 2017 +0800
Committer: Roger Shi
Committed: Wed Aug 30 16:02:26 2017 +0800
--
.../table/DEFAULT.TEST_KYLIN_FACT.json | 2 +-
.../apache/kylin/query/ITKylinQueryTest.java| 5 +
.../test/resources/query/sql_join/query_00.sql | 21
3 files changed, 27 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb3617c/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index e82d535..4be05f2 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -44,7 +44,7 @@
}, {
"id" : "8",
"name" : "SELLER_ID",
-"datatype" : "bigint",
+"datatype" : "int",
"data_gen" : "RAND||1000|10001000",
"index": "T"
}, {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb3617c/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 4ea4497..e332cda 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -234,6 +234,11 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
+public void testJoinCastQuery() throws Exception {
+execAndCompQuery(getQueryFolderPrefix() +
"src/test/resources/query/sql_join", null, true);
+}
+
+@Test
public void testCachedQuery() throws Exception {
execAndCompQuery(getQueryFolderPrefix() +
"src/test/resources/query/sql_cache", null, true);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb3617c/kylin-it/src/test/resources/query/sql_join/query_00.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_join/query_00.sql
b/kylin-it/src/test/resources/query/sql_join/query_00.sql
new file mode 100644
index 000..024e45d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_join/query_00.sql
@@ -0,0 +1,21 @@
+-- join condition cast type
+select sum(ITEM_COUNT) as ITEM_CNT
+FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
+INNER JOIN TEST_ORDER as TEST_ORDER
+ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
+ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
+ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN EDW.TEST_CAL_DT as TEST_CAL_DT
+ON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT
+INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND
TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN EDW.TEST_SITES as TEST_SITES
+ON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID
+INNER JOIN EDW.TEST_SELLER_TYPE_DIM as TEST_SELLER_TYPE_DIM
+ON TEST_KYLIN_FACT.SLR_SEGMENT_CD = TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD
+INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
+ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
+ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
\ No newline at end of file
[16/50] [abbrv] kylin git commit: minor, special treatment for OLAPContext.allColumns when select star
minor, special treatment for OLAPContext.allColumns when select star
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3200d6a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3200d6a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3200d6a1
Branch: refs/heads/2622-2764
Commit: 3200d6a148d4368801e415f6ae969bbab7c7345e
Parents: 28a07eb
Author: Hongbin Ma
Authored: Sun Aug 27 15:41:22 2017 +0800
Committer: Hongbin Ma
Committed: Mon Aug 28 10:42:57 2017 +0800
--
.../org/apache/kylin/query/relnode/OLAPTableScan.java | 13 +++--
1 file changed, 11 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/3200d6a1/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 7801891..dd2160b 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -210,7 +210,8 @@ public class OLAPTableScan extends TableScan implements
OLAPRel, EnumerableRel {
Preconditions.checkState(columnRowType == null, "OLAPTableScan MUST
NOT be shared by more than one prent");
// create context in case of non-join
-if (implementor.getContext() == null || !(implementor.getParentNode()
instanceof OLAPJoinRel) || implementor.isNewOLAPContextRequired()) {
+if (implementor.getContext() == null || !(implementor.getParentNode()
instanceof OLAPJoinRel)
+|| implementor.isNewOLAPContextRequired()) {
implementor.allocateContext();
}
@@ -227,6 +228,13 @@ public class OLAPTableScan extends TableScan implements
OLAPRel, EnumerableRel {
if (context.firstTableScan == null) {
context.firstTableScan = this;
}
+
+if (implementor.getParentNode() instanceof OLAPToEnumerableConverter) {
+// OLAPToEnumerableConverter on top of table scan, should be a
select * from table
+for (TblColRef tblColRef : columnRowType.getAllColumns()) {
+context.allColumns.add(tblColRef);
+}
+}
}
public String getAlias() {
@@ -288,7 +296,8 @@ public class OLAPTableScan extends TableScan implements
OLAPRel, EnumerableRel {
String execFunction = genExecFunc();
PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(),
this.rowType, pref.preferArray());
-MethodCallExpression exprCall =
Expressions.call(table.getExpression(OLAPTable.class), execFunction,
implementor.getRootExpression(), Expressions.constant(context.id));
+MethodCallExpression exprCall =
Expressions.call(table.getExpression(OLAPTable.class), execFunction,
+implementor.getRootExpression(),
Expressions.constant(context.id));
return implementor.result(physType, Blocks.toBlock(exprCall));
}
[09/50] [abbrv] kylin git commit: KYLIN-2807, enhance validate of datamodel's update
KYLIN-2807, enhance validate of datamodel's update
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d873ca13
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d873ca13
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d873ca13
Branch: refs/heads/2622-2764
Commit: d873ca13aa67a90a99b66d99659ca571a3ab977f
Parents: cd9a515
Author: Cheng Wang
Authored: Thu Aug 24 11:12:50 2017 +0800
Committer: Hongbin Ma
Committed: Thu Aug 24 11:27:44 2017 +0800
--
.../apache/kylin/rest/service/ModelService.java | 123 ++-
.../kylin/rest/service/ModelServiceTest.java| 94 +-
2 files changed, 179 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d873ca13/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 14b2280..82632eb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -22,6 +22,7 @@ import static
org.apache.kylin.rest.controller2.ModelControllerV2.VALID_MODELNAM
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -299,55 +300,112 @@ public class ModelService extends BasicService {
return true;
}
-private boolean checkIfBreakExistingCubes(DataModelDesc dataModelDesc,
String project) throws IOException {
+private List getModelCols(DataModelDesc model) {
+List dimCols = new ArrayList();
+
+List dimensions = model.getDimensions();
+
+for (ModelDimensionDesc dim : dimensions) {
+String table = dim.getTable();
+for (String c : dim.getColumns()) {
+dimCols.add(table + "." + c);
+}
+}
+return dimCols;
+}
+
+private List getModelMeasures(DataModelDesc model) {
+List measures = new ArrayList();
+
+for (String s : model.getMetrics()) {
+measures.add(s);
+}
+return measures;
+}
+
+private Map> getInfluencedCubesByDims(List
dims, List cubes) {
+Map> influencedCubes = new HashMap<>();
+for (CubeInstance cubeInstance : cubes) {
+CubeDesc cubeDesc = cubeInstance.getDescriptor();
+for (TblColRef tblColRef :
cubeDesc.listDimensionColumnsIncludingDerived()) {
+if (dims.contains(tblColRef.getIdentity()))
+continue;
+if (influencedCubes.get(tblColRef.getIdentity()) == null) {
+List candidates = new ArrayList<>();
+candidates.add(cubeInstance.getName());
+influencedCubes.put(tblColRef.getIdentity(), candidates);
+} else
+
influencedCubes.get(tblColRef.getIdentity()).add(cubeInstance.getName());
+}
+}
+return influencedCubes;
+}
+
+private Map>
getInfluencedCubesByMeasures(List allCols, List cubes) {
+Map> influencedCubes = new HashMap<>();
+for (CubeInstance cubeInstance : cubes) {
+CubeDesc cubeDesc = cubeInstance.getDescriptor();
+Set tblColRefs =
Sets.newHashSet(cubeDesc.listAllColumns());
+
tblColRefs.removeAll(cubeDesc.listDimensionColumnsIncludingDerived());
+for (TblColRef tblColRef : tblColRefs) {
+if (allCols.contains(tblColRef.getIdentity()))
+continue;
+if (influencedCubes.get(tblColRef.getIdentity()) == null) {
+List candidates = new ArrayList<>();
+candidates.add(cubeInstance.getName());
+influencedCubes.put(tblColRef.getIdentity(), candidates);
+} else
+
influencedCubes.get(tblColRef.getIdentity()).add(cubeInstance.getName());
+}
+}
+return influencedCubes;
+}
+
+private String checkIfBreakExistingCubes(DataModelDesc dataModelDesc,
String project) throws IOException {
String modelName = dataModelDesc.getName();
List cubes = cubeService.listAllCubes(null, project,
modelName, true);
+DataModelDesc originDataModelDesc = listAllModels(modelName, project,
true).get(0);
+StringBuilder checkRet = new StringBuilder();
if (cubes != null && cubes.size() != 0) {
dataModelDesc.init(getConfig(),
getMetadataManager().getAllTablesMap(dataModelDesc.getProject()),
[27/50] [abbrv] kylin git commit: KYLIN-2773: fix join condition push down issue caused by type casting
KYLIN-2773: fix join condition push down issue caused by type casting
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87f1732b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87f1732b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87f1732b
Branch: refs/heads/2622-2764
Commit: 87f1732b0a26191122788baaa2fc9d97c96210da
Parents: 28ac7d9
Author: Zhong
Authored: Tue Aug 29 17:19:47 2017 +0800
Committer: Roger Shi
Committed: Wed Aug 30 16:02:26 2017 +0800
--
.../calcite/sql2rel/SqlToRelConverter.java | 54
1 file changed, 54 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/87f1732b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index f10ba96..3615df7 100644
---
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -2468,6 +2468,10 @@ public class SqlToRelConverter {
return corr;
}
+if (containOnlyCast(joinCond)) {
+ joinCond = convertCastCondition(joinCond);
+}
+
final Join originalJoin =
(Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel,
joinCond, ImmutableSet.of(), joinType, false);
@@ -2475,6 +2479,56 @@ public class SqlToRelConverter {
return RelOptUtil.pushDownJoinConditions(originalJoin);
}
+ private boolean containOnlyCast(RexNode node) {
+boolean result = true;
+switch (node.getKind()) {
+ case AND:
+ case EQUALS:
+final RexCall call = (RexCall) node;
+List operands = Lists.newArrayList(call.getOperands());
+for (int i = 0; i < operands.size(); i++) {
+ RexNode operand = operands.get(i);
+ result &= containOnlyCast(operand);
+}
+break;
+ case OR:
+ case INPUT_REF:
+ case LITERAL:
+ case CAST:
+return true;
+ default:
+return false;
+}
+return result;
+ }
+
+ private static RexNode convertCastCondition(RexNode node) {
+switch (node.getKind()) {
+ case AND:
+ case EQUALS:
+RexCall call = (RexCall) node;
+List list = Lists.newArrayList();
+List operands = Lists.newArrayList(call.getOperands());
+for (int i = 0; i < operands.size(); i++) {
+ RexNode operand = operands.get(i);
+ final RexNode e =
+ convertCastCondition(
+ operand);
+ list.add(e);
+}
+if (!list.equals(call.getOperands())) {
+ return call.clone(call.getType(), list);
+}
+return call;
+ case CAST:
+call = (RexCall) node;
+operands = Lists.newArrayList(call.getOperands());
+return operands.get(0);
+ default:
+return node;
+}
+ }
+
private CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0) {
final Set correlatedVariables =
RelOptUtil.getVariablesUsed(r0);
[17/50] [abbrv] kylin git commit: minor, avoid buffer size overflow
minor, avoid buffer size overflow
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fad75c93
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fad75c93
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fad75c93
Branch: refs/heads/2622-2764
Commit: fad75c93f42d252dd1ee2ec7b03df716c15fb2f4
Parents: 3200d6a
Author: Hongbin Ma
Authored: Mon Aug 28 12:18:23 2017 +0800
Committer: Hongbin Ma
Committed: Mon Aug 28 12:25:12 2017 +0800
--
.../kylin/metadata/filter/TupleFilterSerializer.java | 13 +
1 file changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fad75c93/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index 63153ef..d434f0b 100755
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -69,8 +69,11 @@ public class TupleFilterSerializer {
internalSerialize(rootFilter, decorator, buffer, cs);
break;
} catch (BufferOverflowException e) {
-logger.info("Buffer size {} cannot hold the filter, resizing
to 4 times", bufferSize);
-bufferSize *= 4;
+if (bufferSize == (1 << 30))
+throw e;
+
+logger.info("Buffer size {} cannot hold the filter, resizing
to 2 times", bufferSize);
+bufferSize = bufferSize << 1;
}
}
byte[] result = new byte[buffer.position()];
@@ -78,7 +81,8 @@ public class TupleFilterSerializer {
return result;
}
-private static void internalSerialize(TupleFilter filter, Decorator
decorator, ByteBuffer buffer, IFilterCodeSystem cs) {
+private static void internalSerialize(TupleFilter filter, Decorator
decorator, ByteBuffer buffer,
+IFilterCodeSystem cs) {
if (decorator != null) { // give decorator a chance to manipulate the
output filter
filter = decorator.onSerialize(filter);
}
@@ -203,7 +207,8 @@ public class TupleFilterSerializer {
if (extendedTupleFilters.containsKey(op)) {
try {
filter = (TupleFilter)
extendedTupleFilters.get(op).getConstructor().newInstance();
-} catch (InstantiationException | IllegalAccessException |
InvocationTargetException | NoSuchMethodException e) {
+} catch (InstantiationException | IllegalAccessException |
InvocationTargetException
+| NoSuchMethodException e) {
throw new RuntimeException(e);
}
} else {
[03/50] [abbrv] kylin git commit: KYLIN-2803 allow non-select query when pushdown is enabled
KYLIN-2803 allow non-select query when pushdown is enabled
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f26b1abf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f26b1abf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f26b1abf
Branch: refs/heads/2622-2764
Commit: f26b1abf7cc3052dae937713f2ebec083e185ae2
Parents: a637d2a
Author: shaofengshi
Authored: Tue Aug 22 17:09:13 2017 +0800
Committer: Hongbin Ma
Committed: Tue Aug 22 17:39:02 2017 +0800
--
.../src/main/java/org/apache/kylin/rest/service/QueryService.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f26b1abf/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 75b968b..24ac5a0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -366,7 +366,8 @@ public class QueryService extends BasicService {
logger.info("Using project: " + project);
logger.info("The original query: " + sql);
-if (!sql.toLowerCase().contains("select")) {
+if (!sql.toLowerCase().contains("select")
+&& KylinConfig.getInstanceFromEnv().isPushDownEnabled() ==
false) {
logger.debug("Directly return exception as not supported");
throw new BadRequestException(msg.getNOT_SUPPORTED_SQL());
}
[43/50] [abbrv] kylin git commit: KYLIN-2838 Should get storageType in changeHtableHost of CubeMigrationCLI
KYLIN-2838 Should get storageType in changeHtableHost of CubeMigrationCLI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78543d6e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78543d6e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78543d6e
Branch: refs/heads/2622-2764
Commit: 78543d6e970cfb9dc85bcc48775681afcdb1c0e9
Parents: 34596d3
Author: kangkaisen
Authored: Sun Sep 3 17:52:18 2017 +0800
Committer: kangkaisen
Committed: Sun Sep 3 17:53:37 2017 +0800
--
tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/78543d6e/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index bb308e2..d3865c6 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -201,7 +201,7 @@ public class CubeMigrationCLI {
}
protected void changeHtableHost(CubeInstance cube) {
-if (cube.getDescriptor().getEngineType() !=
IStorageAware.ID_SHARDED_HBASE)
+if (cube.getDescriptor().getStorageType() !=
IStorageAware.ID_SHARDED_HBASE)
return;
for (CubeSegment segment : cube.getSegments()) {
operations
[01/50] [abbrv] kylin git commit: minor, add logs [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/2622-2764 2607e18b5 -> ccd076861 (forced update)
minor, add logs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a637d2a0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a637d2a0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a637d2a0
Branch: refs/heads/2622-2764
Commit: a637d2a0cea88d85a372bc449269a50a1502471f
Parents: 68ae79a
Author: Cheng Wang
Authored: Tue Aug 22 16:18:17 2017 +0800
Committer: æ
Committed: Tue Aug 22 16:46:53 2017 +0800
--
.../main/java/org/apache/kylin/rest/service/CubeService.java| 1 +
.../kylin/storage/hbase/util/HBaseRegionSizeCalculator.java | 5 +++--
2 files changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a637d2a0/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index f8ca38d..fe8fb6c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -718,6 +718,7 @@ public class CubeService extends BasicService implements
InitializingBean {
@Override
public void afterPropertiesSet() throws Exception {
Broadcaster.getInstance(getConfig()).registerListener(new
HTableInfoSyncListener(), "cube");
+logger.info("HTableInfoSyncListener is on.");
}
private class HTableInfoSyncListener extends Broadcaster.Listener {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a637d2a0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 1daca0a..5189628 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -104,8 +104,9 @@ public class HBaseRegionSizeCalculator {
sizeMap.put(regionId, regionSizeBytes);
countMap.put(regionId, new
Pair<>(regionLoad.getStores(), regionLoad.getStorefiles()));
-// logger.info("Region " + regionLoad.getNameAsString()
-// + " has size " + regionSizeBytes);
+if (regionSizeBytes == 0L) {
+logger.info("Region " +
regionLoad.getNameAsString() + " has size " + regionSizeBytes);
+}
}
}
}
[39/50] [abbrv] kylin git commit: KYLIN-2753 Fix for the special case that jobs not really stopped
KYLIN-2753 Fix for the special case that jobs not really stopped
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1004a853
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1004a853
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1004a853
Branch: refs/heads/2622-2764
Commit: 1004a853225503e00f48a27949d7e09109e937db
Parents: 7a47e94
Author: zhengdong
Authored: Fri Sep 1 17:58:21 2017 +0800
Committer: Billy Liu
Committed: Fri Sep 1 18:10:00 2017 +0800
--
.../apache/kylin/job/execution/ExecutableManager.java| 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1004a853/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 124a5a9..0439ddd 100755
---
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -338,12 +338,13 @@ public class ExecutableManager {
break;
}
}
-info = Maps.newHashMap();
final long endTime = job.getEndTime();
-long interruptTime = System.currentTimeMillis() - endTime +
job.getInterruptTime();
-info.putAll(getJobOutput(jobId).getInfo());
-info.put(AbstractExecutable.INTERRUPT_TIME,
Long.toString(interruptTime));
-info.remove(AbstractExecutable.END_TIME);
+if (endTime != 0) {
+long interruptTime = System.currentTimeMillis() - endTime +
job.getInterruptTime();
+info = Maps.newHashMap(getJobOutput(jobId).getInfo());
+info.put(AbstractExecutable.INTERRUPT_TIME,
Long.toString(interruptTime));
+info.remove(AbstractExecutable.END_TIME);
+}
}
updateJobOutput(jobId, ExecutableState.READY, info, null);
}
[47/50] [abbrv] kylin git commit: KYLIN-2833 Storage cleanup job could delete the intermediate hive table used by running jobs
KYLIN-2833 Storage cleanup job could delete the intermediate hive table used by
running jobs
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/14058bda
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/14058bda
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/14058bda
Branch: refs/heads/2622-2764
Commit: 14058bda896f7670c333ca14f614e7cdab0ce62c
Parents: 1da269d
Author: zhengdong
Authored: Fri Sep 1 18:27:57 2017 +0800
Committer: Li Yang
Committed: Sun Sep 3 20:15:58 2017 +0800
--
.../main/java/org/apache/kylin/rest/job/StorageCleanupJob.java | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/14058bda/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
old mode 100644
new mode 100755
index 8bb06b5..6d1d44c
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -324,10 +324,8 @@ public class StorageCleanupJob extends AbstractApplication
{
for (String jobId : workingJobList) {
String segmentId = getSegmentIdFromJobId(jobId);
-if (null == segmentId)
-continue;
-
-return segUuid.equals(segmentId);
+if (segUuid.equals(segmentId))
+return true;
}
return false;
}
[44/50] [abbrv] kylin git commit: KYLIN-2814 fix HTTP connection may not be released in RestClient
KYLIN-2814 fix HTTP connection may not be released in RestClient
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae311b10
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae311b10
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae311b10
Branch: refs/heads/2622-2764
Commit: ae311b10e29f64c519140e4c02d903987981733a
Parents: 78543d6
Author: lichao
Authored: Fri Aug 25 14:31:17 2017 +0800
Committer: Li Yang
Committed: Sun Sep 3 17:57:25 2017 +0800
--
.../java/org/apache/kylin/common/restclient/RestClient.java| 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae311b10/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 13490cb..b0ca5f7 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -121,8 +121,9 @@ public class RestClient {
String url = baseUrl + "/cache/" + entity + "/" + cacheKey + "/" +
event;
HttpPut request = new HttpPut(url);
+HttpResponse response = null;
try {
-HttpResponse response = client.execute(request);
+response = client.execute(request);
if (response.getStatusLine().getStatusCode() != 200) {
String msg = EntityUtils.toString(response.getEntity());
@@ -131,6 +132,9 @@ public class RestClient {
} catch (Exception ex) {
throw new IOException(ex);
} finally {
+if (response != null) {
+EntityUtils.consume(response.getEntity());
+}
request.releaseConnection();
}
}
[31/50] [abbrv] kylin git commit: minor, refactor pushdown code
minor, refactor pushdown code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/068baf6b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/068baf6b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/068baf6b
Branch: refs/heads/2622-2764
Commit: 068baf6b3ec13fe6e68e9a41593dcd139b4a8597
Parents: a4aeb77
Author: Li Yang
Authored: Wed Aug 30 21:18:19 2017 +0800
Committer: Roger Shi
Committed: Thu Aug 31 10:37:32 2017 +0800
--
.../kylin/common/debug/BackdoorToggles.java | 19 +++-
.../source/adhocquery/IPushDownRunner.java | 2 +-
.../org/apache/kylin/query/KylinTestBase.java | 11 +-
.../query/adhoc/PushDownRunnerJdbcImpl.java | 5 +-
.../apache/kylin/query/relnode/OLAPContext.java | 11 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 1 +
.../relnode/OLAPToEnumerableConverter.java | 61 ++-
.../routing/RoutingIndicatorException.java | 35 +++
.../apache/kylin/query/util/PushDownUtil.java | 104 ++-
.../apache/kylin/rest/service/QueryService.java | 37 ---
10 files changed, 145 insertions(+), 141 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/068baf6b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 8cb48b6..88c7c16 100644
---
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -40,6 +40,14 @@ public class BackdoorToggles {
_backdoorToggles.set(toggles);
}
+public static void addToggle(String key, String value) {
+Map map = _backdoorToggles.get();
+if (map == null) {
+setToggles(Maps. newHashMap());
+}
+_backdoorToggles.get().put(key, value);
+}
+
public static void addToggles(Map toggles) {
Map map = _backdoorToggles.get();
if (map == null) {
@@ -47,6 +55,15 @@ public class BackdoorToggles {
}
_backdoorToggles.get().putAll(toggles);
}
+
+// try avoid using this generic method
+public static String getToggle(String key) {
+Map map = _backdoorToggles.get();
+if (map == null)
+return null;
+
+return map.get(key);
+}
public static String getCoprocessorBehavior() {
return getString(DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR);
@@ -109,7 +126,7 @@ public class BackdoorToggles {
public static boolean getPrepareOnly() {
return getBoolean(DEBUG_TOGGLE_PREPARE_ONLY);
}
-
+
private static String getString(String key) {
Map toggles = _backdoorToggles.get();
if (toggles == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/068baf6b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
index 0336bfb..9983f5c 100644
---
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
+++
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
@@ -47,6 +47,6 @@ public interface IPushDownRunner {
*
* @throws Exception if running pushdown fails
*/
-boolean executeUpdate(String sql) throws Exception;
+void executeUpdate(String sql) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/068baf6b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 8f143c7..d3db995 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -70,7 +70,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Lists;
import com.google.common.io.Files;
/**
@@ -261,14 +260,12 @@ public class KylinTestBase {
return output(resultSet, needDisplay);
} catch (SQLException sqlException) {
-List> results = Lists.newArrayList();
-List columnMetas = Lists.newArrayList();
-boolean b
[02/50] [abbrv] kylin git commit: minor, remove user/ADMIN
minor, remove user/ADMIN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/891caabd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/891caabd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/891caabd
Branch: refs/heads/2622-2764
Commit: 891caabd7563b230855d251f4896fdfa9c09d0cd
Parents: aaf9416
Author: Hongbin Ma
Authored: Tue Aug 22 17:26:52 2017 +0800
Committer: Hongbin Ma
Committed: Tue Aug 22 17:27:57 2017 +0800
--
examples/sample_cube/template/user/ADMIN | 19 ---
1 file changed, 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/891caabd/examples/sample_cube/template/user/ADMIN
--
diff --git a/examples/sample_cube/template/user/ADMIN
b/examples/sample_cube/template/user/ADMIN
deleted file mode 100644
index f27a36f..000
--- a/examples/sample_cube/template/user/ADMIN
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "username" : "ADMIN",
- "password" : "skippped-ldap",
- "authorities" : [ {
-"authority" : "ROLE_ADMIN"
- }, {
-"authority" : "ROLE_ANALYST"
- }, {
-"authority" : "ROLE_MODELER"
- } ],
- "disabled" : false,
- "defaultPassword" : false,
- "locked" : false,
- "lockedTime" : 0,
- "wrongTime" : 0,
- "uuid" : null,
- "last_modified" : 0,
- "version" : "2.1.0"
-}
\ No newline at end of file
[13/50] [abbrv] kylin git commit: KYLIN-2811, refine spark cubing
KYLIN-2811, refine spark cubing
Signed-off-by: Hongbin Ma
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1c234a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1c234a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1c234a9
Branch: refs/heads/2622-2764
Commit: a1c234a9afbc5a30306f4275127649f980ab75bd
Parents: 4316cfd
Author: Cheng Wang
Authored: Sun Aug 27 13:29:12 2017 +0800
Committer: Hongbin Ma
Committed: Sun Aug 27 15:08:18 2017 +0800
--
.../org/apache/kylin/common/KylinConfig.java| 50 +--
.../metadata/model/ComputedColumnDesc.java | 10 +-
.../kylin/engine/mr/common/BatchConstants.java | 1 +
.../spark/SparkBatchCubingJobBuilder2.java | 1 +
.../kylin/engine/spark/SparkCubingByLayer.java | 380 ---
.../localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
.../kylin/rest/util/Log4jConfigListener.java| 1 +
7 files changed, 267 insertions(+), 178 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1c234a9/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index a56e9b8..1d5e0ec 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -18,6 +18,15 @@
package org.apache.kylin.common;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.restclient.RestClient;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.OrderedProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -34,26 +43,15 @@ import java.util.Enumeration;
import java.util.Map;
import java.util.Properties;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.restclient.RestClient;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.OrderedProperties;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
/**
*/
public class KylinConfig extends KylinConfigBase {
private static final long serialVersionUID = 1L;
private static final Logger logger =
LoggerFactory.getLogger(KylinConfig.class);
-/** Kylin properties file name */
+/**
+ * Kylin properties file name
+ */
public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties";
public static final String KYLIN_CONF = "KYLIN_CONF";
@@ -62,7 +60,7 @@ public class KylinConfig extends KylinConfigBase {
// thread-local instances, will override SYS_ENV_INSTANCE
private static transient ThreadLocal THREAD_ENV_INSTANCE =
new ThreadLocal<>();
-
+
static {
/*
* Make Calcite to work with Unicode.
@@ -121,9 +119,6 @@ public class KylinConfig extends KylinConfigBase {
}
private static UriType decideUriType(String metaUri) {
-if (metaUri.indexOf("@hdfs") > 0) {
-return UriType.HDFS_FILE;
-}
try {
File file = new File(metaUri);
@@ -163,23 +158,6 @@ public class KylinConfig extends KylinConfigBase {
*/
UriType uriType = decideUriType(uri);
-if (uriType == UriType.HDFS_FILE) {
-KylinConfig config;
-FileSystem fs;
-int cut = uri.indexOf('@');
-String realHdfsPath = uri.substring(0, cut) + "/" +
KYLIN_CONF_PROPERTIES_FILE;
-try {
-config = new KylinConfig();
-fs = HadoopUtil.getFileSystem(realHdfsPath);
-InputStream is = fs.open(new Path(realHdfsPath));
-Properties prop = streamToProps(is);
-config.reloadKylinConfig(prop);
-} catch (IOException e) {
-throw new RuntimeException(e);
-}
-return config;
-}
-
if (uriType == UriType.LOCAL_FOLDER) {
KylinConfig config = new KylinConfig();
config.setMetadataUrl(uri);
@@ -241,7 +219,7 @@ public class KylinConfig extends KylinConfigBase {
public static void setKylinConfigThreadLocal(KylinConfig config) {
THREAD_ENV_INSTANCE.set(config);
}
-
+
public static void removeKylinConfigThrea
[36/50] [abbrv] kylin git commit: KYLIN-2824 Parse Boolean type in JDBC driver
KYLIN-2824 Parse Boolean type in JDBC driver
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a47e947
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a47e947
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a47e947
Branch: refs/heads/2622-2764
Commit: 7a47e9479f7a8c53d284d3bd95cf11b145cf9c36
Parents: 363f858
Author: Tianhao Zhao
Authored: Thu Aug 31 16:10:28 2017 +0800
Committer: Billy Liu
Committed: Fri Sep 1 17:37:31 2017 +0800
--
jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a47e947/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index d42dbb1..cd7b921 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -112,6 +112,7 @@ public class KylinClient implements IRemoteClient {
result = BigDecimal.class;
break;
case Types.BIT:
+case Types.BOOLEAN:
result = Boolean.class;
break;
case Types.TINYINT:
@@ -167,6 +168,7 @@ public class KylinClient implements IRemoteClient {
case Types.DECIMAL:
return new BigDecimal(value);
case Types.BIT:
+case Types.BOOLEAN:
return Boolean.parseBoolean(value);
case Types.TINYINT:
return Byte.valueOf(value);
[20/50] [abbrv] kylin git commit: KYLIN-2816 wrap columns with backtick
KYLIN-2816 wrap columns with backtick
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74c904e4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74c904e4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74c904e4
Branch: refs/heads/2622-2764
Commit: 74c904e4eedc224b07c9b6463ab0b63faddd4a21
Parents: b778f33
Author: tttMelody <245915...@qq.com>
Authored: Mon Aug 28 14:19:36 2017 +0800
Committer: Hongbin Ma
Committed: Mon Aug 28 15:25:27 2017 +0800
--
.../org/apache/kylin/job/JoinedFlatTable.java | 77 ++--
.../kylin/job/TestWrapColumnWithBacktick.java | 57 +++
.../metadata/model/tool/CalciteParser.java | 8 +-
3 files changed, 135 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/74c904e4/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 824c693..4ac5a3a 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -21,11 +21,17 @@ package org.apache.kylin.job;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -35,6 +41,9 @@ import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
@@ -43,6 +52,7 @@ import org.apache.kylin.metadata.model.TblColRef;
public class JoinedFlatTable {
public static final String BACKTICK = "`";
+private static final Logger logger =
LoggerFactory.getLogger(JoinedFlatTable.class);
public static String getTableDir(IJoinedFlatTableDesc flatDesc, String
storageDfsDir) {
return storageDfsDir + "/" + flatDesc.getTableName();
@@ -130,11 +140,7 @@ public class JoinedFlatTable {
sql.append(",");
}
String colTotalName = String.format("%s.%s",
col.getTableRef().getTableName(), col.getName());
-String expressionInSourceDB = col.getExpressionInSourceDB();
-if (expressionInSourceDB.contains(".")) {
-// surround column name with back-tick, to support unicode
column name
-expressionInSourceDB = expressionInSourceDB.replace(".", "." +
BACKTICK) + BACKTICK;
-}
+String expressionInSourceDB =
quoteOriginalColumnWithBacktick(col.getExpressionInSourceDB());
if (skipAsList.contains(colTotalName)) {
sql.append(expressionInSourceDB + sep);
} else {
@@ -277,4 +283,65 @@ public class JoinedFlatTable {
return sql.toString();
}
+/**
+ * quote column name with back-tick, to support unicode column name
+ *
+ * @param sourceSQL
+ * @return
+ */
+static String quoteOriginalColumnWithBacktick(String sourceSQL) {
+StringBuilder result = new StringBuilder(sourceSQL);
+try {
+List> replacePoses = new ArrayList<>();
+for (SqlIdentifier id :
SqlIdentifierVisitor.getAllSqlIdentifier(CalciteParser.getExpNode(sourceSQL))) {
+
replacePoses.add(CalciteParser.getReplacePos(id.getComponentParserPosition(1),
"select " + sourceSQL + " from t"));
+}
+
+// latter replace position in the front of the list.
+Collections.sort(replacePoses, new Comparator>() {
+@Override
+public int compare(Pair o1, Pair o2) {
+return -(o1.getSecond() - o2.getSecond());
+}
+});
+
+// cuz the pos is add "select "'s size, so minus 7 offset.
+final int OFFSET = 7;
+for (Pair replacePos : replacePoses) {
+result.insert(replacePos.getSecond() - OFFSET, "`");
+result.insert(replacePos.getFirst() - OFFSET, "`");
+}
+} catch (
[12/50] [abbrv] kylin git commit: KYLIN-2808, fix table not fould exception
KYLIN-2808, fix table not fould exception
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ff4f634f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ff4f634f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ff4f634f
Branch: refs/heads/2622-2764
Commit: ff4f634fe5ad5f96b1506a2ab368eb701f62c5dc
Parents: 1fff421
Author: Cheng Wang
Authored: Fri Aug 25 19:10:19 2017 +0800
Committer: æ
Committed: Sat Aug 26 00:19:08 2017 +0800
--
.../apache/kylin/source/hive/HiveMRInput.java | 41 +---
1 file changed, 26 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff4f634f/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 71a7152..096134c 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -62,10 +62,12 @@ public class HiveMRInput implements IMRInput {
@SuppressWarnings("unused")
private static final Logger logger =
LoggerFactory.getLogger(HiveMRInput.class);
-
+
public static String getTableNameForHCat(TableDesc table) {
String tableName = (table.isView()) ? table.getMaterializedName() :
table.getName();
-return String.format("%s.%s", table.getDatabase(),
tableName).toUpperCase();
+String database = (table.isView()) ?
KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable()
+: table.getDatabase();
+return String.format("%s.%s", database, tableName).toUpperCase();
}
@Override
@@ -139,17 +141,18 @@ public class HiveMRInput implements IMRInput {
@Override
public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable
jobFlow) {
final String cubeName =
CubingExecutableUtil.getCubeName(jobFlow.getParams());
-final KylinConfig cubeConfig =
CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName).getConfig();
+final KylinConfig cubeConfig =
CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName)
+.getConfig();
final String hiveInitStatements =
JoinedFlatTable.generateHiveInitStatements(flatTableDatabase);
// create flat table first
addStepPhase1_DoCreateFlatTable(jobFlow);
-
+
// then count and redistribute
if (cubeConfig.isHiveRedistributeEnabled()) {
jobFlow.addTask(createRedistributeFlatHiveTableStep(hiveInitStatements,
cubeName));
}
-
+
// special for hive
addStepPhase1_DoMaterializeLookupTable(jobFlow);
}
@@ -158,14 +161,14 @@ public class HiveMRInput implements IMRInput {
final String cubeName =
CubingExecutableUtil.getCubeName(jobFlow.getParams());
final String hiveInitStatements =
JoinedFlatTable.generateHiveInitStatements(flatTableDatabase);
final String jobWorkingDir = getJobWorkingDir(jobFlow);
-
+
jobFlow.addTask(createFlatHiveTableStep(hiveInitStatements,
jobWorkingDir, cubeName));
}
protected void
addStepPhase1_DoMaterializeLookupTable(DefaultChainedExecutable jobFlow) {
final String hiveInitStatements =
JoinedFlatTable.generateHiveInitStatements(flatTableDatabase);
final String jobWorkingDir = getJobWorkingDir(jobFlow);
-
+
AbstractExecutable task =
createLookupHiveViewMaterializationStep(hiveInitStatements, jobWorkingDir);
if (task != null) {
jobFlow.addTask(task);
@@ -186,7 +189,8 @@ public class HiveMRInput implements IMRInput {
return step;
}
-private ShellExecutable createLookupHiveViewMaterializationStep(String
hiveInitStatements, String jobWorkingDir) {
+private ShellExecutable createLookupHiveViewMaterializationStep(String
hiveInitStatements,
+String jobWorkingDir) {
ShellExecutable step = new ShellExecutable();
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP);
@@ -215,21 +219,25 @@ public class HiveMRInput implements IMRInput {
if (lookUpTableDesc.isView()) {
StringBuilder createIntermediateTableHql = new
StringBuilder();
createIntermediateTableHql.append("DROP TABLE IF EXISTS "
+ intermediate + ";\n");
-createIntermediateTable
[41/50] [abbrv] kylin git commit: KYLIN-2818 Refactor dateRange & sourceOffset on CubeSegment
KYLIN-2818 Refactor dateRange & sourceOffset on CubeSegment
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75fbdcff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75fbdcff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75fbdcff
Branch: refs/heads/2622-2764
Commit: 75fbdcffb0366cb555783a229d1a514d010abd87
Parents: a54f016
Author: Li Yang
Authored: Fri Sep 1 15:18:36 2017 +0800
Committer: Hongbin Ma
Committed: Sat Sep 2 18:47:28 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4 +
.../org/apache/kylin/common/util/ClassUtil.java | 1 -
.../main/resources/kylin-defaults.properties| 1 +
.../org/apache/kylin/cube/CubeInstance.java | 12 +-
.../java/org/apache/kylin/cube/CubeManager.java | 164 ++
.../java/org/apache/kylin/cube/CubeSegment.java | 136 ++--
.../apache/kylin/cube/CubeSegmentAdvisor.java | 95
.../org/apache/kylin/cube/CubeValidator.java| 132 ---
.../cube/gridtable/SegmentGTStartAndEnd.java| 11 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../cube/model/CubeJoinedFlatTableDesc.java | 14 +-
.../cube/model/CubeJoinedFlatTableEnrich.java | 18 +-
.../org/apache/kylin/cube/CubeManagerTest.java | 49 +++--
.../org/apache/kylin/cube/CubeSegmentsTest.java | 52 ++---
.../org/apache/kylin/job/JoinedFlatTable.java | 8 +-
.../metadata/model/ComputedColumnDesc.java | 1 +
.../metadata/model/IJoinedFlatTableDesc.java| 5 +-
.../apache/kylin/metadata/model/ISegment.java | 21 +-
.../kylin/metadata/model/ISegmentAdvisor.java | 35 +++
.../apache/kylin/metadata/model/JoinDesc.java | 7 +-
.../kylin/metadata/model/MeasureDesc.java | 1 +
.../kylin/metadata/model/PartitionDesc.java | 89 +---
.../kylin/metadata/model/SegmentRange.java | 220 +++
.../apache/kylin/metadata/model/Segments.java | 166 +++---
.../apache/kylin/metadata/model/TableRef.java | 1 +
.../apache/kylin/metadata/model/TblColRef.java | 2 +-
.../apache/kylin/source/SourcePartition.java| 57 ++---
.../DefaultPartitionConditionBuilderTest.java | 20 +-
.../kylin/storage/translate/HBaseKeyRange.java | 2 +-
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 7 +-
.../engine/mr/steps/MergeCuboidMapperTest.java | 3 +-
.../kylin/provision/BuildCubeWithEngine.java| 14 +-
.../kylin/provision/BuildCubeWithStream.java| 19 +-
.../.settings/org.eclipse.core.resources.prefs | 1 -
.../kylin/rest/controller/CubeController.java | 40 ++--
.../rest/controller2/CubeControllerV2.java | 40 ++--
.../apache/kylin/rest/job/HybridCubeCLI.java| 16 +-
.../apache/kylin/rest/service/CubeService.java | 13 +-
.../apache/kylin/rest/service/JobService.java | 58 ++---
.../apache/kylin/rest/service/ModelService.java | 10 +-
.../rest/controller/CubeControllerTest.java | 10 +-
.../kylin/rest/service/CacheServiceTest.java| 3 +-
.../apache/kylin/source/hive/HiveSource.java| 3 +-
.../apache/kylin/source/jdbc/JdbcSource.java| 3 +-
.../apache/kylin/source/kafka/KafkaSource.java | 22 +-
.../kylin/source/kafka/job/MergeOffsetStep.java | 18 +-
.../hbase/util/ExtendCubeToHybridCLI.java | 6 +-
.../kylin/tool/ExtendCubeToHybridCLI.java | 6 +-
48 files changed, 965 insertions(+), 653 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/75fbdcff/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e5fc3d6..9533400 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -335,6 +335,10 @@ abstract public class KylinConfigBase implements
Serializable {
return getOptional("kylin.cube.cuboid-scheduler",
"org.apache.kylin.cube.cuboid.DefaultCuboidScheduler");
}
+public String getSegmentAdvisor() {
+return getOptional("kylin.cube.segment-advisor",
"org.apache.kylin.cube.CubeSegmentAdvisor");
+}
+
public double getJobCuboidSizeRatio() {
return
Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/75fbdcff/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
b/core-common/src/main/java/org/apache/kylin/common/u
[04/50] [abbrv] kylin git commit: minor, only apply adjustSqlDigest to involved aggregations
minor, only apply adjustSqlDigest to involved aggregations
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d85f6ab0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d85f6ab0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d85f6ab0
Branch: refs/heads/2622-2764
Commit: d85f6ab0130e3c1962c9dc770e06f178d3d85c32
Parents: f26b1ab
Author: Roger Shi
Authored: Tue Aug 22 17:42:37 2017 +0800
Committer: Roger Shi
Committed: Tue Aug 22 17:55:53 2017 +0800
--
.../org/apache/kylin/cube/CubeCapabilityChecker.java | 5 +
.../extendedcolumn/ExtendedColumnMeasureType.java| 10 +-
.../org/apache/kylin/measure/raw/RawMeasureType.java | 10 +-
.../apache/kylin/measure/topn/TopNMeasureType.java | 15 ++-
.../kylin/metadata/realization/CapabilityResult.java | 8
.../apache/kylin/metadata/realization/SQLDigest.java | 10 --
.../apache/kylin/storage/hbase/ITStorageTest.java| 4 +++-
.../kylin/query/enumerator/OLAPEnumerator.java | 1 -
.../org/apache/kylin/query/relnode/OLAPContext.java | 5 -
.../org/apache/kylin/query/routing/QueryRouter.java | 6 ++
10 files changed, 66 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 61516d2..47f268a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -127,6 +127,11 @@ public class CubeCapabilityChecker {
public double suggestCostMultiplier() {
return 100;
}
+
+@Override
+public MeasureDesc getInvolvedMeasure() {
+return null;
+}
});
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
index de5ee25..c95cd1c 100644
---
a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
+++
b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
@@ -95,6 +95,9 @@ public class ExtendedColumnMeasureType extends
MeasureType {
@Override
public void adjustSqlDigest(List measureDescs, SQLDigest
sqlDigest) {
for (MeasureDesc measureDesc : measureDescs) {
+if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
+continue;
+}
FunctionDesc extendColumnFunc = measureDesc.getFunction();
List hosts = getExtendedColumnHosts(extendColumnFunc);
TblColRef extended = getExtendedColumn(extendColumnFunc);
@@ -111,7 +114,7 @@ public class ExtendedColumnMeasureType extends
MeasureType {
}
@Override
-public CapabilityResult.CapabilityInfluence
influenceCapabilityCheck(Collection unmatchedDimensions,
Collection unmatchedAggregations, SQLDigest digest, MeasureDesc
measureDesc) {
+public CapabilityResult.CapabilityInfluence
influenceCapabilityCheck(Collection unmatchedDimensions,
Collection unmatchedAggregations, SQLDigest digest, final
MeasureDesc measureDesc) {
TblColRef extendedCol = getExtendedColumn(measureDesc.getFunction());
if (!unmatchedDimensions.contains(extendedCol)) {
@@ -129,6 +132,11 @@ public class ExtendedColumnMeasureType extends
MeasureType {
public double suggestCostMultiplier() {
return 0.9;
}
+
+@Override
+public MeasureDesc getInvolvedMeasure() {
+return measureDesc;
+}
};
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 31f35e9..eee8fa6 100644
---
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++
b
[21/50] [abbrv] kylin git commit: minor, fix some error messages.
minor, fix some error messages.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/480592d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/480592d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/480592d4
Branch: refs/heads/2622-2764
Commit: 480592d49b8479d4cbba74e889eb5e256d9b2ef8
Parents: 74c904e
Author: Roger Shi
Authored: Mon Aug 28 15:41:59 2017 +0800
Committer: Roger Shi
Committed: Mon Aug 28 15:47:38 2017 +0800
--
server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java | 2 +-
server-base/src/main/java/org/apache/kylin/rest/msg/Message.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/480592d4/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
--
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
index ece4b83..2b1bf8e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
@@ -187,7 +187,7 @@ public class CnMessage extends Message {
}
public String getUPDATE_MODEL_KEY_FIELD() {
-return "已使ç¨ç维度å度éä¸ç°æçè¿æ¥æ
ä¸è½è¢«ä¿®æ¹";
+return "ç±äºç»´åº¦ã度éæè
è¿æ¥å
³ç³»è¢«ä¿®æ¹å¯¼è´ä¸åå¨çcubeå®ä¹ä¸ä¸è´ï¼å èå½å模åæ
æ³ä¿åã";
}
public String getBROKEN_MODEL_DESC() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/480592d4/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
--
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
index 8137c34..146a0a5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
@@ -187,7 +187,7 @@ public class Message {
}
public String getUPDATE_MODEL_KEY_FIELD() {
-return "Dimensions and measures in use and existing join tree cannot
be modified.";
+return "Model cannot save because there are dimensions, measures or
join relations modified to be inconsistent with existing cube.";
}
public String getBROKEN_MODEL_DESC() {
[37/50] [abbrv] kylin git commit: minor, code clean up in KylinConfig
ontributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.Properties;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ * @author kangkaisen
+ */
+
+public class HotLoadKylinPropertiesTestCase extends LocalFileMetadataTestCase {
+@Before
+public void setUp() throws Exception {
+this.createTestMetadata();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
+
+protected void updateProperty(String key, String value) {
+File propFile = KylinConfig.getSitePropertiesFile();
+Properties conf = new Properties();
+
+//load
+try (FileInputStream is = new FileInputStream(propFile)) {
+conf.load(is);
+conf.setProperty(key, value);
+} catch (Exception e) {
+System.err.println(e.getMessage());
+}
+
+//store
+try (FileOutputStream out = new FileOutputStream(propFile)) {
+conf.store(out, null);
+} catch (Exception e) {
+System.err.println(e.getMessage());
+}
+}
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index f20ed7c..6da854b 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -27,7 +27,6 @@ import static org.junit.Assert.assertTrue;
import java.util.Map;
import java.util.Properties;
-import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
import org.junit.Test;
import com.google.common.collect.Maps;
@@ -85,7 +84,7 @@ public class KylinConfigTest extends
HotLoadKylinPropertiesTestCase {
assertEquals("who...@kylin.apache.org", config.getKylinOwner());
updateProperty("kylin.storage.hbase.owner-tag",
"ky...@kylin.apache.org");
-KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+KylinConfig.getInstanceFromEnv().reloadFromSiteProperties();
assertEquals("ky...@kylin.apache.org", config.getKylinOwner());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
b/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
deleted file mode 100644
index 9f5b278..000
---
a/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import org.apache.kylin.common.KylinConfig;
-import org.junit.After;
-import org.junit.Before;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import
[06/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as default metric framework
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java
--
diff --git
a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java
b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java
new file mode 100644
index 000..5e3e00b
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.metrics;
+
+import static
org.apache.kylin.common.metrics.common.Metricss.buildCubeMetricPrefix;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+import org.apache.kylin.common.metrics.common.Metrics;
+import org.apache.kylin.common.metrics.common.MetricsConstant;
+import org.apache.kylin.common.metrics.common.MetricsFactory;
+import org.apache.kylin.common.metrics.common.Metricss;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.rest.service.ServiceTestBase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.codahale.metrics.Snapshot;
+
+public class QueryMetrics2Test extends ServiceTestBase {
+
+private static MBeanServer mBeanServer;
+private static ObjectName objectName;
+
+@Before
+public void setup() throws Exception {
+super.setup();
+
+objectName = new ObjectName("Hadoop:service=Kylin,name=Server_Total");
+}
+
+@Test
+public void testQueryMetrics() throws Exception {
+System.setProperty("kylin.server.query-metrics-enabled", "true");
+QueryMetrics2Facade.init();
+
+SQLRequest sqlRequest = new SQLRequest();
+sqlRequest.setSql("select * from TEST_KYLIN_FACT");
+sqlRequest.setProject("default");
+
+SQLResponse sqlResponse = new SQLResponse();
+sqlResponse.setDuration(10);
+sqlResponse.setCube("test_cube");
+sqlResponse.setIsException(false);
+sqlResponse.setTotalScanCount(100);
+List list1 = new ArrayList<>();
+list1.add("111");
+list1.add("112");
+List list2 = new ArrayList<>();
+list2.add("111");
+list2.add("112");
+List> results = new ArrayList<>();
+results.add(list1);
+results.add(list2);
+sqlResponse.setResults(results);
+sqlResponse.setStorageCacheUsed(true);
+Metrics metrics = MetricsFactory.getInstance();
+QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
+String prefix = buildCubeMetricPrefix(sqlRequest.getProject(),
sqlResponse.getCube().replace("=", "->"));
+Thread.sleep(2000);
+
+Assert.assertEquals(1L,
+metrics.getCounter(Metricss.buildMetricName(prefix,
MetricsConstant.QUERY_COUNT)).getCount());
+Assert.assertEquals(1L,
+metrics.getCounter(Metricss.buildMetricName(prefix,
MetricsConstant.QUERY_SUCCESS_COUNT)).getCount());
+Assert.assertEquals(0L,
+metrics.getCounter(Metricss.buildMetricName(prefix,
MetricsConstant.QUERY_FAIL_COUNT)).getCount());
+Assert.assertEquals(1L,
+metrics.getCounter(Metricss.buildMetricName(prefix,
MetricsConstant.QUERY_CACHE_COUNT)).getCount());
+//
+Snapshot queryScanSnapshot= metrics
+.getHistogram(Metricss.buildMetricName(prefix,
MetricsConstant.QUERY_SCAN_ROWCOUNT)).getSnapshot();
+Assert.assertEquals(100.0, queryScanSnapshot.getMean(), 0);
+Assert.assertEquals(100.0, queryScanSnapshot.getMax(), 0);
+Assert.assertEquals(100.0, queryScanSnapshot.getMin(), 0);
+
+Snapshot queryResultSnapshot = metrics
+.getHistogram(Metricss.buildMetricName(prefix,
MetricsConstant.QUERY_RESULT_ROWCOUNT)).getSnapshot();
+
+Assert.assertEquals(2.0, queryResultSnapshot.getMean(), 0);
+Assert.assertEquals(2.0, queryResultSnapshot.getMax(), 0);
[45/50] [abbrv] kylin git commit: KYLIN-2814 code cleanup in RestClient
KYLIN-2814 code cleanup in RestClient
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e4474e4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e4474e4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e4474e4
Branch: refs/heads/2622-2764
Commit: 2e4474e46cb563096635d0b749ee023e0fcb66d0
Parents: ae311b1
Author: Li Yang
Authored: Sun Sep 3 18:38:15 2017 +0800
Committer: Li Yang
Committed: Sun Sep 3 18:38:15 2017 +0800
--
.../kylin/common/restclient/RestClient.java | 144 ---
1 file changed, 90 insertions(+), 54 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e4474e4/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index b0ca5f7..95aa626 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -27,7 +27,8 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import javax.xml.bind.DatatypeConverter;
+
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -45,20 +46,17 @@ import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.apache.kylin.common.util.JsonUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import javax.xml.bind.DatatypeConverter;
+import com.fasterxml.jackson.databind.ObjectMapper;
/**
* @author yangli9
*/
public class RestClient {
-protected String host;
-protected int port;
-protected String baseUrl;
-protected String userName;
-protected String password;
-protected DefaultHttpClient client;
+private static final Logger logger =
LoggerFactory.getLogger(RestClient.class);
protected static Pattern fullRestPattern =
Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
@@ -74,9 +72,17 @@ public class RestClient {
return m.matches();
}
+//
+
+protected String host;
+protected int port;
+protected String baseUrl;
+protected String userName;
+protected String password;
+protected DefaultHttpClient client;
+
/**
- * @param uri
- *"user:pwd@host:port"
+ * @param uri "user:pwd@host:port"
*/
public RestClient(String uri) {
Matcher m = fullRestPattern.matcher(uri);
@@ -127,32 +133,30 @@ public class RestClient {
if (response.getStatusLine().getStatusCode() != 200) {
String msg = EntityUtils.toString(response.getEntity());
-throw new IOException("Invalid response " +
response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n"
+ msg);
+throw new IOException("Invalid response " +
response.getStatusLine().getStatusCode()
++ " with cache wipe url " + url + "\n" + msg);
}
-} catch (Exception ex) {
-throw new IOException(ex);
} finally {
-if (response != null) {
-EntityUtils.consume(response.getEntity());
-}
-request.releaseConnection();
+cleanup(request, response);
}
}
public String getKylinProperties() throws IOException {
String url = baseUrl + "/admin/config";
HttpGet request = new HttpGet(url);
+HttpResponse response = null;
try {
-HttpResponse response = client.execute(request);
+response = client.execute(request);
String msg = EntityUtils.toString(response.getEntity());
Map map = JsonUtil.readValueAsMap(msg);
msg = map.get("config");
if (response.getStatusLine().getStatusCode() != 200)
-throw new IOException("Invalid response " +
response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n"
+ msg);
+throw new IOException("Invalid response " +
response.getStatusLine().getStatusCode()
++ " with cache wipe url " + url + "\n" + msg);
return msg;
} finally {
-request.releaseConnection();
+cleanup(request, response);
}
}
@@ -167,18 +171,24 @@ public class RestClient {
public boolean
[49/50] [abbrv] kylin git commit: KYLIN-2622 Add SegmentAppendTrieDictBuilder for count distinct measure
KYLIN-2622 Add SegmentAppendTrieDictBuilder for count distinct measure
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8de0d529
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8de0d529
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8de0d529
Branch: refs/heads/2622-2764
Commit: 8de0d529723eca45fb2aeca5789a5d8f8dc8edb6
Parents: 520f627
Author: kangkaisen
Authored: Tue May 16 17:04:55 2017 +0800
Committer: kangkaisen
Committed: Mon Sep 4 21:38:46 2017 +0800
--
.../apache/kylin/dict/AppendTrieDictionary.java | 9 ++-
.../apache/kylin/dict/DictionaryManager.java| 10 +++
.../kylin/dict/GlobalDictionaryBuilder.java | 8 +--
.../global/AppendTrieDictionaryBuilder.java | 15 ++--
.../kylin/dict/global/GlobalDictHDFSStore.java | 54 +-
.../kylin/dict/global/GlobalDictMetadata.java | 1 -
.../kylin/dict/global/GlobalDictStore.java | 5 +-
.../global/SegmentAppendTrieDictBuilder.java| 76
.../dict/global/AppendTrieDictionaryTest.java | 3 +-
.../kylin/measure/bitmap/BitmapMeasureType.java | 21 +-
.../localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
webapp/app/js/model/cubeConfig.js | 3 +-
.../cubeDesigner/advanced_settings.html | 13 +++-
13 files changed, 162 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8de0d529/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index b9f0d2b..18795b4 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -38,8 +38,8 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.Objects;
+import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
-import static com.google.common.base.Preconditions.checkState;
/**
* A dictionary based on Trie data structure that maps enumerations of byte[]
to
@@ -72,7 +72,12 @@ public class AppendTrieDictionary extends
CacheDictionary {
this.baseDir = baseDir;
final GlobalDictStore globalDictStore = new
GlobalDictHDFSStore(baseDir);
Long[] versions = globalDictStore.listAllVersions();
-checkState(versions.length > 0, "Global dict at %s is empty", baseDir);
+
+if (versions.length == 0) {
+this.metadata = new GlobalDictMetadata(0, 0, 0, 0, null, new
TreeMap());
+return; // for the removed SegmentAppendTrieDictBuilder
+}
+
final long latestVersion = versions[versions.length - 1];
final Path latestVersionPath =
globalDictStore.getVersionDir(latestVersion);
this.metadata = globalDictStore.getMetadata(latestVersion);
http://git-wip-us.apache.org/repos/asf/kylin/blob/8de0d529/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 857ee30..47e5c29 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -227,6 +227,16 @@ public class DictionaryManager {
if (dicts.size() == 1)
return dicts.get(0);
+/**
+ * AppendTrieDictionary needn't merge
+ * more than one AppendTrieDictionary will generate when user use
{@link SegmentAppendTrieDictBuilder}
+ */
+for (DictionaryInfo dict: dicts) {
+if
(dict.getDictionaryClass().equals(AppendTrieDictionary.class.getName())) {
+return dict;
+}
+}
+
DictionaryInfo firstDictInfo = null;
int totalSize = 0;
for (DictionaryInfo info : dicts) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/8de0d529/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
index a593371..404d53c 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
+++
b/core-dictionary/src/main/java/org
[15/50] [abbrv] kylin git commit: KYLIN-2816 support hive unicode column name
KYLIN-2816 support hive unicode column name
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28a07ebd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28a07ebd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28a07ebd
Branch: refs/heads/2622-2764
Commit: 28a07ebd3983c325c64c0642e7dedbf32c284f39
Parents: e2e0102
Author: Billy(Yiming) Liu
Authored: Sun Aug 27 18:55:03 2017 +0800
Committer: GitHub
Committed: Sun Aug 27 18:55:03 2017 +0800
--
.../kylin/common/util/HiveCmdBuilder.java | 5 ++--
.../kylin/common/util/HiveCmdBuilderTest.java | 2 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 28
.../apache/kylin/source/hive/HiveMRInput.java | 2 +-
4 files changed, 28 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 2f6b9a0..191bbc0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -66,11 +66,12 @@ public class HiveCmdBuilder {
switch (clientMode) {
case CLI:
-buf.append("hive -e \"");
+// use single-quote to ignore the executing back-ticks surrounded
column name in shell
+buf.append("hive -e \'");
for (String statement : statements) {
buf.append(statement).append("\n");
}
-buf.append("\"");
+buf.append("\'");
buf.append(parseProps());
break;
case BEELINE:
http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index d69d4d2..d2938b5 100644
---
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -60,7 +60,7 @@ public class HiveCmdBuilderTest {
hiveCmdBuilder.addStatement("SHOW\n TABLES;");
hiveCmdBuilder.setHiveConfProps(hiveProps);
hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite);
-assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n
TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
+assertEquals("hive -e \'USE default;\nDROP TABLE test;\nSHOW\n
TABLES;\n\' --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index acb29e1..824c693 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -42,6 +42,8 @@ import org.apache.kylin.metadata.model.TblColRef;
public class JoinedFlatTable {
+public static final String BACKTICK = "`";
+
public static String getTableDir(IJoinedFlatTableDesc flatDesc, String
storageDfsDir) {
return storageDfsDir + "/" + flatDesc.getTableName();
}
@@ -75,7 +77,7 @@ public class JoinedFlatTable {
ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
}
ddl.append("STORED AS " + format + "\n");
-ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) +
"';").append("\n");
+ddl.append("LOCATION '\"'\"'" + getTableDir(flatDesc, storageDfsDir) +
"'\"'\"';").append("\n");
return ddl.toString();
}
@@ -128,10 +130,15 @@ public class JoinedFlatTable {
sql.append(",");
}
String colTotalName = String.format("%s.%s",
col.getTableRef().getTableName(), col.getName());
+String expressionInSourceDB = col.getExpressionInSourceDB();
+if (expressionInSourceDB.contains(".")) {
+// surround column name with back-tick, to support unicode
column name
+expressionInSourceDB = expressionInSourceDB.replace(".", "." +
BACKTICK)
[34/50] [abbrv] kylin git commit: Revert "KYLIN-2790 Add extending point for column family"
Revert "KYLIN-2790 Add extending point for column family"
This reverts commit 842dc4ca1dabb576aed40c55ebfc1cbabaaf3944.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/363f858b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/363f858b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/363f858b
Branch: refs/heads/2622-2764
Commit: 363f858b890ea04c87ee7b081fc74805766e0b93
Parents: 32d346b
Author: Hongbin Ma
Authored: Thu Aug 31 22:43:36 2017 +0800
Committer: Hongbin Ma
Committed: Thu Aug 31 22:43:36 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4
.../org/apache/kylin/cube/model/CubeDesc.java | 18 ++--
.../kylin/cube/model/HBaseMappingDesc.java | 18
.../org/apache/kylin/gridtable/GTRecord.java| 22
4 files changed, 2 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/363f858b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d32f08c..0e990df 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -286,10 +286,6 @@ abstract public class KylinConfigBase implements
Serializable {
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}
-
-public String getHBaseMappingAdapter() {
-return getOptional("kylin.metadata.hbasemapping-adapter");
-}
//
// DICTIONARY & SNAPSHOT
http://git-wip-us.apache.org/repos/asf/kylin/blob/363f858b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index d263e1f..0e22587 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -22,7 +22,6 @@ import static
com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
-import java.lang.reflect.Method;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
@@ -604,21 +603,8 @@ public class CubeDesc extends RootPersistentEntity
implements IEngineAware {
validateAggregationGroups(); // check if aggregation group is valid
validateAggregationGroupsCombination();
-String hbaseMappingAdapterName = config.getHBaseMappingAdapter();
-
-if (hbaseMappingAdapterName != null) {
-try {
-Class hbaseMappingAdapterClass =
Class.forName(hbaseMappingAdapterName);
-Method initMethod =
hbaseMappingAdapterClass.getMethod("initHBaseMapping", CubeDesc.class);
-initMethod.invoke(null, this);
-} catch (Exception e) {
-logger.error("Wrong configuration for
kylin.metadata.hbasemapping-adapter: class "
-+ hbaseMappingAdapterName + " not found. ");
-}
-} else {
-if (hbaseMapping != null) {
-hbaseMapping.init(this);
-}
+if (hbaseMapping != null) {
+hbaseMapping.init(this);
}
initMeasureReferenceToColumnFamily();
http://git-wip-us.apache.org/repos/asf/kylin/blob/363f858b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index 77b4b14..d1e5829 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -87,24 +87,6 @@ public class HBaseMappingDesc implements
java.io.Serializable {
}
}
-public void initAsSeparatedColumns(CubeDesc cubeDesc) {
-cubeRef = cubeDesc;
-
-int cfNum = cubeDesc.getMeasures().size();
-columnFamily = new HBaseColumnFamilyDesc[cfNum];
-
-for (int i = 0; i < cfN
[10/50] [abbrv] kylin git commit: KYLIN-2809 update calcite to support "+" as string concat
KYLIN-2809 update calcite to support "+" as string concat Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d059c05e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d059c05e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d059c05e Branch: refs/heads/2622-2764 Commit: d059c05e7a064c6c6bc861fb3e399985ed968177 Parents: d873ca1 Author: Roger Shi Authored: Thu Aug 24 15:28:10 2017 +0800 Committer: Roger Shi Committed: Thu Aug 24 15:46:12 2017 +0800 -- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/d059c05e/pom.xml -- diff --git a/pom.xml b/pom.xml index 5b00798..d82a7f4 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ -1.13.0-kylin-r1-SNAPSHOT +1.13.0-kylin-r2-SNAPSHOT 1.10.0
[08/50] [abbrv] kylin git commit: minor, fix flanky test query
minor, fix flanky test query Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd9a5152 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd9a5152 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd9a5152 Branch: refs/heads/2622-2764 Commit: cd9a515286eb7ba0553339cfb98d86ea10377a90 Parents: 8e07884 Author: Hongbin Ma Authored: Wed Aug 23 14:05:47 2017 +0800 Committer: Hongbin Ma Committed: Thu Aug 24 10:30:24 2017 +0800 -- kylin-it/src/test/resources/query/sql/query98.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/cd9a5152/kylin-it/src/test/resources/query/sql/query98.sql -- diff --git a/kylin-it/src/test/resources/query/sql/query98.sql b/kylin-it/src/test/resources/query/sql/query98.sql index d73a4cb..81736d9 100644 --- a/kylin-it/src/test/resources/query/sql/query98.sql +++ b/kylin-it/src/test/resources/query/sql/query98.sql @@ -16,6 +16,6 @@ -- limitations under the License. -- -select cal_dt, sum(price) as x,leaf_categ_id as GMV +select cal_dt,leaf_categ_id as x, sum(price) as GMV from test_kylin_fact group by leaf_categ_id, cal_dt
[32/50] [abbrv] kylin git commit: Merge commit '727920b4d3642aaa3657d90b7f3dce7dcdd68fe2'
Merge commit '727920b4d3642aaa3657d90b7f3dce7dcdd68fe2' Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b50030da Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b50030da Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b50030da Branch: refs/heads/2622-2764 Commit: b50030da4cd64d424a82e9d2b752c00917e8e681 Parents: 068baf6 727920b Author: Hongbin Ma Authored: Thu Aug 31 15:42:16 2017 +0800 Committer: Hongbin Ma Committed: Thu Aug 31 21:33:14 2017 +0800 -- assembly/pom.xml| 2 +- atopcalcite/pom.xml | 2 +- core-common/pom.xml | 2 +- .../org/apache/kylin/common/KylinConfig.java| 57 ++- .../apache/kylin/common/KylinConfigBase.java| 1 + .../org/apache/kylin/common/KylinConfigExt.java | 2 +- .../org/apache/kylin/common/KylinVersion.java | 4 +- .../org/apache/kylin/common/util/ClassUtil.java | 13 +- core-cube/pom.xml | 2 +- .../kylin/cube/gridtable/CubeCodeSystem.java| 4 + .../org/apache/kylin/gridtable/GTRecord.java| 8 + .../kylin/gridtable/GTSampleCodeSystem.java | 4 + .../apache/kylin/gridtable/IGTCodeSystem.java | 3 + core-dictionary/pom.xml | 2 +- core-job/pom.xml| 2 +- .../job/execution/DefaultChainedExecutable.java | 13 +- .../kylin/job/execution/ExecutableManager.java | 12 +- core-metadata/pom.xml | 2 +- .../measure/bitmap/BitmapCounterFactory.java| 2 + .../kylin/measure/bitmap/BitmapSerializer.java | 38 +- .../measure/bitmap/RoaringBitmapCounter.java| 10 + .../bitmap/RoaringBitmapCounterFactory.java | 5 + .../measure/percentile/PercentileCounter.java | 28 +- .../metadata/datatype/DataTypeSerializer.java | 13 + .../percentile/PercentileCounterTest.java | 46 --- core-storage/pom.xml| 2 +- .../gtrecord/GTCubeStorageQueryBase.java| 53 ++- .../gtrecord/SequentialCubeTupleIterator.java | 17 +- engine-mr/pom.xml | 2 +- .../engine/mr/common/AbstractHadoopJob.java | 128 +++--- .../kylin/engine/mr/common/BatchConstants.java | 1 + .../kylin/engine/mr/common/HadoopCmdOutput.java | 19 +- .../engine/mr/common/JobInfoConverter.java | 7 +- .../engine/mr/common/JobRelatedMetaUtil.java| 72 engine-spark/pom.xml| 2 +- .../engine/spark/KylinKryoRegistrator.java | 11 +- .../spark/SparkBatchCubingJobBuilder2.java | 5 + .../kylin/engine/spark/SparkCubingByLayer.java | 392 +++ .../kylin/engine/spark/SparkExecutable.java | 58 ++- .../spark/util/PercentileCounterSerializer.java | 55 +++ .../spark/util/PercentileSerializerTest.java| 67 .../acl/0928468a-9fab-4185-9a14-6f2e7c74823f| 27 ++ .../acl/2fbca32a-a33e-4b69-83dd-0bb8b1f8c53b| 27 ++ .../acl/2fbca32a-a33e-4b69-83dd-0bb8b1f8c91b| 24 ++ jdbc/pom.xml| 2 +- kylin-it/pom.xml| 18 +- .../job/ITDistributedSchedulerBaseTest.java | 2 + .../jdbc/ITJdbcSourceTableLoaderTest.java | 114 ++ .../source/jdbc/ITJdbcTableReaderTest.java | 109 ++ .../test/resources/query/sql_limit/query05.sql | 21 + pom.xml | 2 +- query/pom.xml | 2 +- .../query/adhoc/PushDownRunnerJdbcImpl.java | 2 - .../kylin/query/relnode/OLAPAggregateRel.java | 4 +- .../kylin/query/relnode/OLAPWindowRel.java | 2 - .../kylin/query/routing/RealizationChooser.java | 7 +- server-base/pom.xml | 2 +- .../kylin/rest/controller/CubeController.java | 49 ++- .../rest/controller2/CubeControllerV2.java | 19 +- .../apache/kylin/rest/service/AclService.java | 4 +- .../rest/service/AclTableMigrationTool.java | 13 +- .../apache/kylin/rest/service/JobService.java | 4 + .../apache/kylin/rest/service/QueryService.java | 4 +- .../kylin/rest/util/Log4jConfigListener.java| 1 + server/pom.xml | 2 +- server/src/main/resources/kylinSecurity.xml | 2 + .../kylin/rest/service/JobServiceTest.java | 33 ++ source-hive/pom.xml | 7 +- .../HiveColumnCardinalityUpdateJob.java | 3 +- .../apache/kylin/source/jdbc/JdbcExplorer.java | 6 +- source-kafka/pom.xml| 2 +- .../kylin/source/kafka/DateTimeParser.java | 62 +++ storage-hbase/pom.xml | 2 +- .../kylin/storage/hbase/HBaseConnection.java| 2 +- .../kylin/storage/hbase/HBaseResourceStore.java | 7 +- .../hbas
[14/50] [abbrv] kylin git commit: KYLIN-2812, Fix the wrong destination database during load kafka topic (#2074)
KYLIN-2812, Fix the wrong destination database during load kafka topic (#2074)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2e0102c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2e0102c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2e0102c
Branch: refs/heads/2622-2764
Commit: e2e0102c4bac289c1e0d61226794030fd93cb145
Parents: ff4f634
Author: Billy(Yiming) Liu
Authored: Sun Aug 27 18:51:50 2017 +0800
Committer: GitHub
Committed: Sun Aug 27 18:51:50 2017 +0800
--
.../rest/controller2/StreamingControllerV2.java | 3 +-
.../controller/StreamingControllerTest.java | 52
2 files changed, 53 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e0102c/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
b/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
index 54733ea..90f2a45 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
@@ -247,10 +247,9 @@ public class StreamingControllerV2 extends BasicController
{
}
if (null != desc) {
-String[] dbTable = HadoopUtil.parseHiveTableName(desc.getName());
+String[] dbTable =
HadoopUtil.parseHiveTableName(desc.getIdentity());
desc.setName(dbTable[1]);
desc.setDatabase(dbTable[0]);
-desc.getIdentity();
}
return desc;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e0102c/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java
--
diff --git
a/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java
b/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java
new file mode 100644
index 000..171d22e
--- /dev/null
+++
b/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class StreamingControllerTest {
+
+@Test
+public void testReadTableDesc() throws IOException {
+String requestTableData =
"{\"name\":\"my_table_name\",\"source_type\":1,\"columns\":[{\"id\":1,\"name\":"
+
+
"\"amount\",\"datatype\":\"decimal\"},{\"id\":2,\"name\":\"category\",\"datatype\":\"varchar(256)\"},"
+
+
"{\"id\":3,\"name\":\"order_time\",\"datatype\":\"timestamp\"},{\"id\":4,\"name\":\"device\","
+
+
"\"datatype\":\"varchar(256)\"},{\"id\":5,\"name\":\"qty\",\"datatype\":\"int\"},{\"id\":6,\"name\":"
+
+
"\"user_id\",\"datatype\":\"varchar(256)\"},{\"id\":7,\"name\":\"user_age\",\"datatype\":\"int\"},"
+
+
"{\"id\":8,\"name\":\"user_gender\",\"datatype\":\"varchar(256)\"},{\"id\":9,\"name\":\"currency\","
+
+
"\"datatype\":\"varchar(256)\"},{\"id\":10,\"name\":\"country\",\"datatype\":\"varchar(256)\"},"
+
+
"{\"id\":11,\"name\":\"year_start\",\"datatype\":\"date\"},{\"id\":12,\"name\":\"quarter_start\","
+
+
"\"datatype\":\"date\"},{\"id\":13,\"name\":\"month_start\",\"datatype\":\"date\"},{\"id\":14,"
+
+
"\"name\":\"week_start\",\"datatype\":\"date\"},{\"id\":15,\"name\":\"day_start\",\"datatype\":"
+
+
"\"date\"},{\"id\":16,\"na
[50/50] [abbrv] kylin git commit: KYLIN-2764 Build UHC Dict Use MR
KYLIN-2764 Build UHC Dict Use MR
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ccd07686
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ccd07686
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ccd07686
Branch: refs/heads/2622-2764
Commit: ccd076861415a40926ab42e6f9b71cd5258d4daf
Parents: 8de0d52
Author: kangkaisen
Authored: Fri Jul 21 15:22:30 2017 +0800
Committer: kangkaisen
Committed: Mon Sep 4 21:41:50 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 11 +-
.../java/org/apache/kylin/cube/CubeManager.java | 82 +-
.../apache/kylin/dict/DictionaryGenerator.java | 14 +-
.../org/apache/kylin/dict/DictionaryInfo.java | 4 +
.../kylin/dict/GlobalDictionaryBuilder.java | 9 +-
.../apache/kylin/dict/IDictionaryBuilder.java | 2 +-
.../global/SegmentAppendTrieDictBuilder.java| 4 +-
.../kylin/dict/DictionaryProviderTest.java | 2 +-
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 26 +++-
.../kylin/engine/mr/JobBuilderSupport.java | 24 ++-
.../engine/mr/common/AbstractHadoopJob.java | 3 +
.../kylin/engine/mr/common/BatchConstants.java | 4 +
.../engine/mr/steps/CreateDictionaryJob.java| 16 +-
.../mr/steps/FactDistinctColumnsReducer.java| 2 +-
.../kylin/engine/mr/steps/UHCDictionaryJob.java | 154 +++
.../engine/mr/steps/UHCDictionaryMapper.java| 101
.../mr/steps/UHCDictionaryPartitioner.java | 30
.../engine/mr/steps/UHCDictionaryReducer.java | 113 ++
.../test_case_data/sandbox/kylin.properties | 5 +
.../dict/ITGlobalDictionaryBuilderTest.java | 4 +-
21 files changed, 553 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ccd07686/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 749b515..485bf21 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -899,6 +899,10 @@ abstract public class KylinConfigBase implements
Serializable {
return getPropertiesByPrefix("kylin.engine.mr.config-override.");
}
+public Map getUHCMRConfigOverride() {
+return getPropertiesByPrefix("kylin.engine.mr.uhc-config-override.");
+}
+
public Map getSparkConfigOverride() {
return getPropertiesByPrefix("kylin.engine.spark-conf.");
}
@@ -925,9 +929,14 @@ abstract public class KylinConfigBase implements
Serializable {
//UHC: ultra high cardinality columns, contain the ShardByColumns and the
GlobalDictionaryColumns
public int getUHCReducerCount() {
-return
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
+return
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "5"));
}
+public boolean isBuildUHCDictWithMREnabled() {
+return
Boolean.parseBoolean(getOptional("kylin.engine.mr.build-uhc-dict", "true"));
+}
+
+
public boolean isBuildDictInReducerEnabled() {
return
Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer",
"true"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ccd07686/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 043993c..1ca4b08 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -30,7 +30,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -229,6 +228,7 @@ public class CubeManager implements IRealizationProvider {
return null;
String builderClass = cubeDesc.getDictionaryBuilderClass(col);
+
DictionaryInfo dictInfo =
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable,
builderClass);
@@ -878,6 +878,52 @@ public class CubeManager implements IRealizationProvider {
return factDictCols;
[25/50] [abbrv] kylin git commit: minor change getProject() in DataModelDesc
minor change getProject() in DataModelDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28ac7d9a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28ac7d9a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28ac7d9a
Branch: refs/heads/2622-2764
Commit: 28ac7d9aeeeb8818207a363231b1354652f9aa8d
Parents: 487a854
Author: tttMelody <245915...@qq.com>
Authored: Tue Aug 29 17:22:41 2017 +0800
Committer: Hongbin Ma
Committed: Tue Aug 29 17:40:33 2017 +0800
--
.../java/org/apache/kylin/metadata/model/DataModelDesc.java | 4 ++--
.../org/apache/kylin/rest/controller/ModelController.java| 2 +-
.../java/org/apache/kylin/rest/service/ModelService.java | 8
.../java/org/apache/kylin/rest/service/ModelServiceTest.java | 8
4 files changed, 11 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/28ac7d9a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index a5da833..b27d644 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -854,11 +854,11 @@ public class DataModelDesc extends RootPersistentEntity {
}
public String getProject() {
-return
ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProjectOfModel(this.getName()).getName();
+return
ProjectManager.getInstance(getConfig()).getProjectOfModel(this.getName()).getName();
}
public ProjectInstance getProjectInstance() {
-return
ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProjectOfModel(this.getName());
+return
ProjectManager.getInstance(getConfig()).getProjectOfModel(this.getName());
}
public static DataModelDesc getCopyOf(DataModelDesc orig) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/28ac7d9a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
index 4226e87..c5bee9f 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
@@ -131,7 +131,7 @@ public class ModelController extends BasicController {
return modelRequest;
}
try {
-modelDesc = modelService.updateModelAndDesc(modelDesc);
+modelDesc =
modelService.updateModelAndDesc(modelRequest.getProject(), modelDesc);
} catch (AccessDeniedException accessDeniedException) {
throw new ForbiddenException("You don't have right to update this
model.");
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/28ac7d9a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 82632eb..42cf085 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -143,8 +143,8 @@ public class ModelService extends BasicService {
return createdDesc;
}
-public DataModelDesc updateModelAndDesc(DataModelDesc desc) throws
IOException {
-aclEvaluate.hasProjectWritePermission(desc.getProjectInstance());
+public DataModelDesc updateModelAndDesc(String project, DataModelDesc
desc) throws IOException {
+aclEvaluate.checkProjectWritePermission(project);
getMetadataManager().updateDataModelDesc(desc);
return desc;
}
@@ -428,7 +428,7 @@ public class ModelService extends BasicService {
}
public DataModelDesc updateModelToResourceStore(DataModelDesc modelDesc,
String projectName) throws IOException {
-
aclEvaluate.hasProjectWritePermission(getProjectManager().getProject(projectName));
+aclEvaluate.checkProjectWritePermission(projectName);
Message msg = MsgPicker.getMsg();
modelDesc.setDraft(false);
@@ -445,7 +445,7 @@ public class ModelService extends BasicService {
if (!error.isEm
[30/50] [abbrv] kylin git commit: #949 Refine column family for measures
#949 Refine column family for measures
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a4aeb778
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a4aeb778
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a4aeb778
Branch: refs/heads/2622-2764
Commit: a4aeb77890577e46300bc702c1117bf542081cc9
Parents: d62cb58
Author: auphyroc99 <454530...@qq.com>
Authored: Wed Aug 30 16:03:24 2017 +0800
Committer: Roger Shi
Committed: Wed Aug 30 18:13:52 2017 +0800
--
.../org/apache/kylin/gridtable/GTRecord.java | 19 ++-
1 file changed, 6 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a4aeb778/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 862fb99..c3a160a 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -45,7 +45,7 @@ public class GTRecord implements Comparable,
Cloneable {
}
this.info = info;
}
-
+
@Override
public GTRecord clone() { // deep copy
ByteArray[] cols = new ByteArray[this.cols.length];
@@ -114,7 +114,6 @@ public class GTRecord implements Comparable,
Cloneable {
return result;
}
-
/** decode and return the values of this record */
public Object[] getValues(int[] selectedColumns, Object[] result) {
assert selectedColumns.length <= result.length;
@@ -295,27 +294,21 @@ public class GTRecord implements Comparable,
Cloneable {
buf.position(pos);
}
}
-
-public void loadColumnsFromColumnBlocks(ImmutableBitSet[]
selectedColumnBlocks, ImmutableBitSet selectedCols, ByteBuffer buf) {
-
+
+public void loadColumnsFromColumnBlocks(ImmutableBitSet[]
selectedColumnBlocks, ImmutableBitSet selectedCols,
+ByteBuffer buf) {
int pos = buf.position();
-
for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) {
-
for (int i = 0; i < selectedColBlock.trueBitCount(); i++) {
-
int c = selectedColBlock.trueBitAt(i);
-
int len = info.codeSystem.codeLength(c, buf);
-if(selectedCols.get(c)) {
-cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
+if (selectedCols.get(c)) {
+cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
}
pos += len;
buf.position(pos);
}
-
}
-
}
}
[35/50] [abbrv] kylin git commit: KYLIN-2775 fix wrong tsColName parsing
KYLIN-2775 fix wrong tsColName parsing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7f47b51a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7f47b51a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7f47b51a
Branch: refs/heads/2622-2764
Commit: 7f47b51a3ebe0ccd0283fa79995bfb02f4c3919e
Parents: 363f858
Author: Billy Liu
Authored: Fri Sep 1 13:49:12 2017 +0800
Committer: æ
Committed: Fri Sep 1 14:43:21 2017 +0800
--
.../template/kafka/DEFAULT.KYLIN_STREAMING_TABLE.json | 2 +-
.../java/org/apache/kylin/source/kafka/StreamingParser.java | 2 +-
.../org/apache/kylin/source/kafka/config/KafkaConfig.java | 7 ---
3 files changed, 6 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f47b51a/examples/sample_cube/template/kafka/DEFAULT.KYLIN_STREAMING_TABLE.json
--
diff --git
a/examples/sample_cube/template/kafka/DEFAULT.KYLIN_STREAMING_TABLE.json
b/examples/sample_cube/template/kafka/DEFAULT.KYLIN_STREAMING_TABLE.json
index 6c02b27..02bf0d0 100644
--- a/examples/sample_cube/template/kafka/DEFAULT.KYLIN_STREAMING_TABLE.json
+++ b/examples/sample_cube/template/kafka/DEFAULT.KYLIN_STREAMING_TABLE.json
@@ -6,7 +6,7 @@
"timeout": 6,
"bufferSize": 65536,
"parserName": "org.apache.kylin.source.kafka.TimedJsonStreamParser",
- "parserProperties": "tsColName=order_time",
+ "parserTimeStampField": "order_time",
"last_modified": 0,
"clusters": [
{
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f47b51a/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
--
diff --git
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
index 2e3c11c..c2b5104 100644
---
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
+++
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/StreamingParser.java
@@ -76,7 +76,7 @@ public abstract class StreamingParser {
Constructor constructor = clazz.getConstructor(List.class,
Map.class);
return (StreamingParser) constructor.newInstance(columns,
properties);
} else {
-throw new IllegalStateException("invalid StreamingConfig,
parserName " + parserName + ", parserProperties " + parserProperties + ".");
+throw new IllegalStateException("Invalid StreamingConfig,
parserName " + parserName + ", parserProperties " + parserProperties + ".");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f47b51a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
--
diff --git
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
index 547e738..ce1d706 100644
---
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
+++
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
@@ -34,6 +34,7 @@ import org.apache.kylin.metadata.MetadataConstants;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonManagedReference;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.source.kafka.TimedJsonStreamParser;
/**
*/
@@ -141,12 +142,12 @@ public class KafkaConfig extends RootPersistentEntity {
public String getAllParserProperties() {
StringBuilder sb = new StringBuilder();
-if (parserProperties != null)
-sb.append(parserProperties);
if (parserTimeStampField != null) {
+sb.append(TimedJsonStreamParser.PROPERTY_TS_COLUMN_NAME + "=" +
parserTimeStampField);
sb.append(";");
-sb.append(parserTimeStampField);
}
+if (parserProperties != null)
+sb.append(parserProperties);
return sb.toString();
}
[11/50] [abbrv] kylin git commit: minor, fix some files license header
minor, fix some files license header
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1fff4214
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1fff4214
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1fff4214
Branch: refs/heads/2622-2764
Commit: 1fff42146aa539b599802f9b19de28dc87d05fad
Parents: d059c05
Author: Roger Shi
Authored: Fri Aug 25 17:31:19 2017 +0800
Committer: Roger Shi
Committed: Fri Aug 25 17:36:24 2017 +0800
--
.../calcite/prepare/CalcitePrepareImpl.java | 34
.../calcite/sql2rel/SqlToRelConverter.java | 34
.../java/org/apache/calcite/tools/Programs.java | 34
3 files changed, 42 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1fff4214/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
--
diff --git
a/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
index b1ce7ea..7b4c549 100644
---
a/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
+++
b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
@@ -1,26 +1,20 @@
/*
- * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://kyligence.io
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * This software is the confidential and proprietary information of
- * Kyligence Inc. ("Confidential Information"). You shall not disclose
- * such Confidential Information and shall use it only in accordance
- * with the terms of the license agreement you entered into with
- * Kyligence Inc.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
package org.apache.calcite.prepare;
import static org.apache.calcite.util.Static.RESOURCE;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1fff4214/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 18aadef..f10ba96 100644
---
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -1,26 +1,20 @@
/*
- * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://kyligence.io
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * This software is the confidential and proprietary information of
- * Kyligence Inc. ("Confidential Information"). You shall not disclose
- * such Confidential Information and shall use it only
[24/50] [abbrv] kylin git commit: Revert "KYLIN-2816 wrap columns with backtick"
Revert "KYLIN-2816 wrap columns with backtick"
This reverts commit 53b25c6137731c1b0d26925be7dcaa0252d75b64.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81e9f573
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81e9f573
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81e9f573
Branch: refs/heads/2622-2764
Commit: 81e9f573fae2af9511ac6004f69101598fc7a71a
Parents: dbecb9b
Author: Hongbin Ma
Authored: Mon Aug 28 19:19:10 2017 +0800
Committer: Hongbin Ma
Committed: Tue Aug 29 10:36:10 2017 +0800
--
.../org/apache/kylin/job/JoinedFlatTable.java | 77 ++--
.../kylin/job/TestWrapColumnWithBacktick.java | 57 ---
.../metadata/model/tool/CalciteParser.java | 8 +-
3 files changed, 7 insertions(+), 135 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/81e9f573/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 4ac5a3a..824c693 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -21,17 +21,11 @@ package org.apache.kylin.job;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.calcite.sql.SqlIdentifier;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -41,9 +35,6 @@ import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
*
@@ -52,7 +43,6 @@ import org.slf4j.LoggerFactory;
public class JoinedFlatTable {
public static final String BACKTICK = "`";
-private static final Logger logger =
LoggerFactory.getLogger(JoinedFlatTable.class);
public static String getTableDir(IJoinedFlatTableDesc flatDesc, String
storageDfsDir) {
return storageDfsDir + "/" + flatDesc.getTableName();
@@ -140,7 +130,11 @@ public class JoinedFlatTable {
sql.append(",");
}
String colTotalName = String.format("%s.%s",
col.getTableRef().getTableName(), col.getName());
-String expressionInSourceDB =
quoteOriginalColumnWithBacktick(col.getExpressionInSourceDB());
+String expressionInSourceDB = col.getExpressionInSourceDB();
+if (expressionInSourceDB.contains(".")) {
+// surround column name with back-tick, to support unicode
column name
+expressionInSourceDB = expressionInSourceDB.replace(".", "." +
BACKTICK) + BACKTICK;
+}
if (skipAsList.contains(colTotalName)) {
sql.append(expressionInSourceDB + sep);
} else {
@@ -283,65 +277,4 @@ public class JoinedFlatTable {
return sql.toString();
}
-/**
- * quote column name with back-tick, to support unicode column name
- *
- * @param sourceSQL
- * @return
- */
-static String quoteOriginalColumnWithBacktick(String sourceSQL) {
-StringBuilder result = new StringBuilder(sourceSQL);
-try {
-List> replacePoses = new ArrayList<>();
-for (SqlIdentifier id :
SqlIdentifierVisitor.getAllSqlIdentifier(CalciteParser.getExpNode(sourceSQL))) {
-
replacePoses.add(CalciteParser.getReplacePos(id.getComponentParserPosition(1),
"select " + sourceSQL + " from t"));
-}
-
-// latter replace position in the front of the list.
-Collections.sort(replacePoses, new Comparator>() {
-@Override
-public int compare(Pair o1, Pair o2) {
-return -(o1.getSecond() - o2.getSecond());
-}
-});
-
-// cuz the pos is add "select "'s size, so minus 7 offset.
-final int OFFSET = 7;
-for (Pair replacePos : replacePoses) {
-result.insert(replacePos.getSecond() - OFFSET, "`");
-result.insert(replacePos.getFirst() - OFFSET, "`
[07/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as default metric framework
KYLIN-2776 Using dropwizard as default metric framework
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8e078847
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8e078847
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8e078847
Branch: refs/heads/2622-2764
Commit: 8e078847a42b4f7853a2c1a75afa405c09c4ea7b
Parents: d85f6ab
Author: yiming.xu <100650...@qq.com>
Authored: Wed Aug 23 13:12:58 2017 +0800
Committer: Hongbin Ma
Committed: Wed Aug 23 13:49:23 2017 +0800
--
core-common/pom.xml | 17 +
.../apache/kylin/common/KylinConfigBase.java| 20 +
.../kylin/common/metrics/common/Metrics.java| 162 ++
.../common/metrics/common/MetricsConstant.java | 38 ++
.../common/metrics/common/MetricsFactory.java | 62 +++
.../common/metrics/common/MetricsScope.java | 33 ++
.../common/metrics/common/MetricsVariable.java | 27 +
.../kylin/common/metrics/common/Metricss.java | 40 ++
.../metrics/metrics2/CodahaleMetrics.java | 508 +++
.../metrics/metrics2/CodahaleReporter.java | 31 ++
.../metrics2/ConsoleMetricsReporter.java| 51 ++
.../metrics/metrics2/JmxMetricsReporter.java| 55 ++
.../metrics2/JsonFileMetricsReporter.java | 136 +
.../metrics2/KylinObjectNameFactory.java| 68 +++
.../metrics2/MetricVariableRatioGauge.java | 47 ++
.../metrics/metrics2/Metrics2Reporter.java | 60 +++
.../metrics/metrics2/MetricsReporting.java | 26 +
.../common/metrics/perflog/IPerfLogger.java | 48 ++
.../common/metrics/perflog/PerfLogger.java | 158 ++
.../metrics/perflog/PerfLoggerFactory.java | 56 ++
.../metrics/perflog/SimplePerfLogger.java | 73 +++
pom.xml | 3 +-
.../kylin/rest/init/InitialTaskManager.java | 4 +-
.../kylin/rest/metrics/QueryMetrics2Facade.java | 106
.../apache/kylin/rest/service/QueryService.java | 2 +
.../kylin/rest/metrics/QueryMetrics2Test.java | 128 +
26 files changed, 1957 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 8852743..d58de36 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -68,6 +68,23 @@
junit
test
+
+
+io.dropwizard.metrics
+metrics-jvm
+${dropwizard.version}
+
+
+io.dropwizard.metrics
+metrics-json
+${dropwizard.version}
+
+
+
+com.github.joshelser
+
dropwizard-metrics-hadoop-metrics2-reporter
+0.1.2
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b622825..99b9bb2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1173,4 +1173,24 @@ abstract public class KylinConfigBase implements
Serializable {
public boolean isWebCrossDomainEnabled() {
return
Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
}
+
+/**
+ * metric
+ */
+public String getCoadhaleMetricReportClassesName() {
+return getOptional("kylin.metric.codahale-metric-report-classes",
+
"org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter");
+}
+
+public String getMetricFileLocation() {
+return getOptional("kylin.metric.file.location", "/tmp/report.json");
+}
+
+public Long getJsonFileMetricsReporterInterval() {
+return
Long.parseLong(getOptional("kylin.metric.json-file-metric-reporter.interval",
"5000"));
+}
+
+public String getPerfLoggerClassName() {
+return getOptional("kylin.metric.perf-logger.class",
"org.apache.kylin.common.metrics.perflog.PerfLogger");
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
ne
[18/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as default metric framework modify comment code
KYLIN-2776 Using dropwizard as default metric framework
modify comment code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b778f33a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b778f33a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b778f33a
Branch: refs/heads/2622-2764
Commit: b778f33af3718fd1bf5f705b81104f424803fe78
Parents: fad75c9
Author: yiming.xu <100650...@qq.com>
Authored: Mon Aug 28 12:57:10 2017 +0800
Committer: Hongbin Ma
Committed: Mon Aug 28 13:50:36 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4 ++
.../kylin/common/metrics/common/Metrics.java| 40 ---
.../common/metrics/common/MetricsConstant.java | 2 +-
.../common/metrics/common/MetricsFactory.java | 2 +-
.../metrics/common/MetricsNameBuilder.java | 38 ++
.../common/metrics/common/MetricsScope.java | 8 +--
.../kylin/common/metrics/common/Metricss.java | 40 ---
.../metrics/metrics2/CodahaleMetrics.java | 20
.../metrics2/ConsoleMetricsReporter.java| 2 +-
.../metrics/metrics2/JmxMetricsReporter.java| 2 +-
.../metrics2/MetricVariableRatioGauge.java | 2 +-
.../metrics/metrics2/Metrics2Reporter.java | 2 +-
.../metrics/metrics2/MetricsReporting.java | 3 --
.../common/metrics/perflog/PerfLogger.java | 3 +-
.../kylin/rest/metrics/QueryMetrics2Facade.java | 20
.../kylin/rest/metrics/QueryMetrics2Test.java | 52 ++--
16 files changed, 122 insertions(+), 118 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b778f33a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 99b9bb2..bca00e7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1149,6 +1149,10 @@ abstract public class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(getOptional("kylin.server.query-metrics-enabled",
"false"));
}
+public boolean getQueryMetrics2Enabled() {
+return
Boolean.parseBoolean(getOptional("kylin.server.query-metrics2-enabled",
"false"));
+}
+
public int[] getQueryMetricsPercentilesIntervals() {
String[] dft = { "60", "300", "3600" };
return
getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b778f33a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
index dcda3cd..9bf8e7a 100644
---
a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
+++
b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
@@ -17,13 +17,13 @@
*/
package org.apache.kylin.common.metrics.common;
+import java.util.concurrent.TimeUnit;
+
import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
-import java.util.concurrent.TimeUnit;
-
/**
* Generic Metics interface.
*/
@@ -35,7 +35,6 @@ public interface Metrics {
public void close() throws Exception;
/**
- *
* @param name starts a scope of a given name. Scopes is stored as
thread-local variable.
*/
public void startStoredScope(String name);
@@ -43,12 +42,14 @@ public interface Metrics {
/**
* Closes the stored scope of a given name.
* Note that this must be called on the same thread as where the scope was
started.
+ *
* @param name
*/
public void endStoredScope(String name);
/**
* Create scope with given name and returns it.
+ *
* @param name
* @return
*/
@@ -56,6 +57,7 @@ public interface Metrics {
/**
* Close the given scope.
+ *
* @param scope
*/
public void endScope(MetricsScope scope);
@@ -64,6 +66,7 @@ public interface Metrics {
/**
* Increments a counter of the given name by 1.
+ *
* @param name
* @return
*/
@@ -71,6 +74,7 @@ public interface Metrics {
/**
* Increments a counter of the given name by "increment"
+ *
* @param name
* @param increment
* @return
@@ -79,6 +83,7 @@ public interface
[46/50] [abbrv] kylin git commit: KYLIN-2781 make find-hadoop-conf-dir.sh executable
KYLIN-2781 make find-hadoop-conf-dir.sh executable
Signed-off-by: Li Yang
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1da269d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1da269d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1da269d5
Branch: refs/heads/2622-2764
Commit: 1da269d5912aed83eebb93a2e4429fd025321386
Parents: 2e4474e
Author: yanghao3
Authored: Fri Sep 1 16:09:12 2017 +0800
Committer: Li Yang
Committed: Sun Sep 3 19:51:04 2017 +0800
--
build/bin/find-hadoop-conf-dir.sh | 106 +
1 file changed, 54 insertions(+), 52 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1da269d5/build/bin/find-hadoop-conf-dir.sh
--
diff --git a/build/bin/find-hadoop-conf-dir.sh
b/build/bin/find-hadoop-conf-dir.sh
index 403bc97..f5658ce 100755
--- a/build/bin/find-hadoop-conf-dir.sh
+++ b/build/bin/find-hadoop-conf-dir.sh
@@ -21,57 +21,59 @@ source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
echo Retrieving hadoop conf dir...
-override_hadoop_conf_dir=`bash ${KYLIN_HOME}/bin/get-properties.sh
kylin.env.hadoop-conf-dir`
-
-if [ -n "$override_hadoop_conf_dir" ]; then
-verbose "kylin_hadoop_conf_dir is override as $override_hadoop_conf_dir"
-export kylin_hadoop_conf_dir=${override_hadoop_conf_dir}
-return
-fi
-
-hbase_classpath=`hbase classpath`
-
-arr=(`echo $hbase_classpath | cut -d ":" -f 1- | sed 's/:/ /g'`)
-kylin_hadoop_conf_dir=
-
-for data in ${arr[@]}
-do
-result=`echo $data | grep -v -E ".*jar"`
-if [ $result ]
-then
-valid_conf_dir=true
-
-if [ ! -f $result/yarn-site.xml ]
-then
-verbose "$result is not valid hadoop dir conf because
yarn-site.xml is missing"
-valid_conf_dir=false
-continue
-fi
-
-if [ ! -f $result/mapred-site.xml ]
-then
-verbose "$result is not valid hadoop dir conf because
mapred-site.xml is missing"
-valid_conf_dir=false
-continue
-fi
-
-if [ ! -f $result/hdfs-site.xml ]
-then
-verbose "$result is not valid hadoop dir conf because
hdfs-site.xml is missing"
-valid_conf_dir=false
-continue
-fi
-
-if [ ! -f $result/core-site.xml ]
-then
-verbose "$result is not valid hadoop dir conf because
core-site.xml is missing"
-valid_conf_dir=false
-continue
-fi
-
-verbose "kylin_hadoop_conf_dir is $result"
-export kylin_hadoop_conf_dir=$result
+function find_hadoop_conf_dir() {
+override_hadoop_conf_dir=`bash ${KYLIN_HOME}/bin/get-properties.sh
kylin.env.hadoop-conf-dir`
+
+if [ -n "$override_hadoop_conf_dir" ]; then
+verbose "kylin_hadoop_conf_dir is override as
$override_hadoop_conf_dir"
+export kylin_hadoop_conf_dir=${override_hadoop_conf_dir}
return
fi
-done
-
+
+hbase_classpath=`hbase classpath`
+
+arr=(`echo $hbase_classpath | cut -d ":" -f 1- | sed 's/:/ /g'`)
+kylin_hadoop_conf_dir=
+
+for data in ${arr[@]}
+do
+result=`echo $data | grep -v -E ".*jar"`
+if [ $result ]
+then
+valid_conf_dir=true
+
+if [ ! -f $result/yarn-site.xml ]
+then
+verbose "$result is not valid hadoop dir conf because
yarn-site.xml is missing"
+valid_conf_dir=false
+continue
+fi
+
+if [ ! -f $result/mapred-site.xml ]
+then
+verbose "$result is not valid hadoop dir conf because
mapred-site.xml is missing"
+valid_conf_dir=false
+continue
+fi
+
+if [ ! -f $result/hdfs-site.xml ]
+then
+verbose "$result is not valid hadoop dir conf because
hdfs-site.xml is missing"
+valid_conf_dir=false
+continue
+fi
+
+if [ ! -f $result/core-site.xml ]
+then
+verbose "$result is not valid hadoop dir conf because
core-site.xml is missing"
+valid_conf_dir=false
+continue
+fi
+
+verbose "kylin_hadoop_conf_dir is $result"
+export kylin_hadoop_conf_dir=$result
+return
+fi
+done
+}
+find_hadoop_conf_dir
\ No newline at end of file
[22/50] [abbrv] kylin git commit: KYLIN-2803 Pushdown non select query
KYLIN-2803 Pushdown non select query
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dbecb9b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dbecb9b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dbecb9b8
Branch: refs/heads/2622-2764
Commit: dbecb9b84bc90049acb3ef314128ac71f3516731
Parents: 480592d
Author: shi shaofeng
Authored: Fri Aug 25 16:18:21 2017 +0800
Committer: SHAOFENG SHI
Committed: Mon Aug 28 16:02:43 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4 ++
.../adhocquery/HivePushDownConverter.java | 4 ++
.../source/adhocquery/IPushDownRunner.java | 13 ++
.../query/adhoc/PushDownRunnerJdbcImpl.java | 36 -
.../apache/kylin/query/util/PushDownUtil.java | 32 ++-
.../org/apache/kylin/query/util/QueryUtil.java | 28 +
.../apache/kylin/query/util/QueryUtilTest.java | 39 ++
.../apache/kylin/rest/service/QueryService.java | 42 +---
8 files changed, 172 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index bca00e7..a113327 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1088,6 +1088,10 @@ abstract public class KylinConfigBase implements
Serializable {
new String[] {
"org.apache.kylin.source.adhocquery.HivePushDownConverter" });
}
+public boolean isPushdownQueryCacheEnabled() {
+return
Boolean.parseBoolean(this.getOptional("kylin.query.pushdown.cache-enabled",
"false"));
+}
+
public String getJdbcUrl() {
return getOptional("kylin.query.pushdown.jdbc.url", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
index bcd8608..eef4594 100644
---
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
@@ -262,6 +262,10 @@ public class HivePushDownConverter implements
IPushDownConverter {
// Step7.Add quote for interval in timestampadd
convertedSql = timestampaddReplace(convertedSql);
+// Step8.Replace integer with int
+convertedSql = replaceString(convertedSql, "INTEGER", "INT");
+convertedSql = replaceString(convertedSql, "integer", "int");
+
return convertedSql;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
index c8d18aa..0336bfb 100644
---
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
+++
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
@@ -36,4 +36,17 @@ public interface IPushDownRunner {
* @throws Exception if running pushdown query fails
*/
void executeQuery(String query, List> returnRows,
List returnColumnMeta) throws Exception;
+
+
+/**
+ * Run an pushdown non-query sql
+ *
+ * @param sql the sql statement
+ *
+ * @return whether the SQL is executed successfully
+ *
+ * @throws Exception if running pushdown fails
+ */
+boolean executeUpdate(String sql) throws Exception;
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
index 751e6b0..713629a 100644
---
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
+++
b/query/src/main/java/
[38/50] [abbrv] kylin git commit: minor, add api to get table_ext from table_desc
minor, add api to get table_ext from table_desc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a54f016a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a54f016a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a54f016a
Branch: refs/heads/2622-2764
Commit: a54f016aff86e93652e7df0a401c270ca6795d13
Parents: 4b3a44b
Author: lidongsjtu
Authored: Fri Sep 1 14:44:00 2017 +0800
Committer: Hongbin Ma
Committed: Fri Sep 1 17:53:18 2017 +0800
--
.../src/main/java/org/apache/kylin/metadata/MetadataManager.java | 4
1 file changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a54f016a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 367ad64..b8141fe 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -217,6 +217,10 @@ public class MetadataManager {
if (t == null)
return null;
+return getTableExt(t);
+}
+
+public TableExtDesc getTableExt(TableDesc t) {
TableExtDesc result = srcTableExtMap.get(mapKey(t.getIdentity(),
t.getProject()));
// avoid returning null, since the TableDesc exists
[28/50] [abbrv] kylin git commit: KYLIN-2823 Trim TupleFilter after dictionary-based filter optimization
KYLIN-2823 Trim TupleFilter after dictionary-based filter optimization
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47822891
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47822891
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47822891
Branch: refs/heads/2622-2764
Commit: 47822891332aef146ae3c2b5e3adc98c23024770
Parents: 1eb3617
Author: Hongbin Ma
Authored: Wed Aug 30 16:06:18 2017 +0800
Committer: æ
Committed: Wed Aug 30 17:40:03 2017 +0800
--
.../java/org/apache/kylin/gridtable/GTUtil.java | 24 +++-
.../metadata/filter/ConstantTupleFilter.java| 14 +++
.../filter/FilterOptimizeTransformer.java | 35 +-
.../filter/FilterOptimizeTransformerTest.java | 121 +++
.../storage/gtrecord/DictGridTableTest.java | 108 +++--
.../src/test/resources/query/sql/query106.sql | 101
.../src/test/resources/query/sql/query107.sql | 102
7 files changed, 459 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/47822891/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 7a7e4e6..5c9dfe3 100755
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -30,6 +30,7 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
@@ -64,7 +65,15 @@ public class GTUtil {
public static TupleFilter convertFilterColumnsAndConstants(TupleFilter
rootFilter, GTInfo info, //
List colMapping, Set
unevaluatableColumnCollector) {
Map map = colListToMap(colMapping);
-return convertFilter(rootFilter, info, map, true,
unevaluatableColumnCollector);
+TupleFilter filter = convertFilter(rootFilter, info, map, true,
unevaluatableColumnCollector);
+
+// optimize the filter: after translating with dictionary, some
filters become determined
+// e.g.
+// ( a = 'value_in_dict' OR a = 'value_not_in_dict') will become (a =
'value_in_dict' OR ConstantTupleFilter.FALSE)
+// use the following to further trim the filter to (a =
'value_in_dict')
+// The goal is to avoid too many children after flatten filter step
+filter = new FilterOptimizeTransformer().transform(filter);
+return filter;
}
protected static Map colListToMap(List
colMapping) {
@@ -81,8 +90,9 @@ public class GTUtil {
final Set unevaluatableColumnCollector) {
IFilterCodeSystem filterCodeSystem =
wrap(info.codeSystem.getComparator());
-
-GTConvertDecorator decorator = new
GTConvertDecorator(unevaluatableColumnCollector, colMapping, info,
encodeConstants);
+
+GTConvertDecorator decorator = new
GTConvertDecorator(unevaluatableColumnCollector, colMapping, info,
+encodeConstants);
byte[] bytes = TupleFilterSerializer.serialize(rootFilter, decorator,
filterCodeSystem);
return TupleFilterSerializer.deserialize(bytes, filterCodeSystem);
@@ -122,14 +132,15 @@ public class GTUtil {
protected final GTInfo info;
protected final boolean encodeConstants;
-public GTConvertDecorator(Set unevaluatableColumnCollector,
Map colMapping, GTInfo info, boolean encodeConstants) {
+public GTConvertDecorator(Set unevaluatableColumnCollector,
Map colMapping,
+GTInfo info, boolean encodeConstants) {
this.unevaluatableColumnCollector = unevaluatableColumnCollector;
this.colMapping = colMapping;
this.info = info;
this.encodeConstants = encodeConstants;
buf = ByteBuffer.allocate(info.getMaxColumnLength());
}
-
+
protected int mapCol(TblColRef col) {
Integer i = colMapping.get(col);
return i == null ? -1 : i;
@@ -143,7 +154,8 @@ public class GTUtil {
// In case of NOT(unEvaluatableFilter), we should immediately
replace it as TRUE,
// Otherwise, unEvaluatableFilter will later be replace with TRUE
and NOT(unEvaluatableFilter)
// will always return F
[19/50] [abbrv] kylin git commit: KYLIN-2811, fix NPE
KYLIN-2811, fix NPE
Signed-off-by: Hongbin Ma
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/727920b4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/727920b4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/727920b4
Branch: refs/heads/2622-2764
Commit: 727920b4d3642aaa3657d90b7f3dce7dcdd68fe2
Parents: a1c234a
Author: Cheng Wang
Authored: Mon Aug 28 12:47:58 2017 +0800
Committer: Hongbin Ma
Committed: Mon Aug 28 14:41:26 2017 +0800
--
.../kylin/engine/spark/SparkCubingByLayer.java | 43 +++-
1 file changed, 24 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/727920b4/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index a3a6ad0..a03e238 100644
---
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -100,7 +100,6 @@ public class SparkCubingByLayer extends AbstractApplication
implements Serializa
.withDescription("Configuration Path").create("confPath");
private Options options;
-private static String metaUrl;
public SparkCubingByLayer() {
options = new Options();
@@ -117,7 +116,7 @@ public class SparkCubingByLayer extends AbstractApplication
implements Serializa
return options;
}
-public static KylinConfig getKylinConfigForExecutor() {
+public static KylinConfig getKylinConfigForExecutor(String metaUrl) {
File file = new
File(SparkFiles.get(KylinConfig.KYLIN_CONF_PROPERTIES_FILE));
String confPath = file.getParentFile().getAbsolutePath();
System.setProperty(KylinConfig.KYLIN_CONF, confPath);
@@ -128,7 +127,7 @@ public class SparkCubingByLayer extends AbstractApplication
implements Serializa
@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
-metaUrl = optionsHelper.getOptionValue(OPTION_META_URL);
+String metaUrl = optionsHelper.getOptionValue(OPTION_META_URL);
String hiveTable = optionsHelper.getOptionValue(OPTION_INPUT_TABLE);
String cubeName = optionsHelper.getOptionValue(OPTION_CUBE_NAME);
String segmentId = optionsHelper.getOptionValue(OPTION_SEGMENT_ID);
@@ -178,17 +177,17 @@ public class SparkCubingByLayer extends
AbstractApplication implements Serializa
// encode with dimension encoding, transform to
RDD
final JavaPairRDD encodedBaseRDD =
intermediateTable.javaRDD()
-.mapToPair(new EncodeBaseCuboid(cubeName, segmentId));
+.mapToPair(new EncodeBaseCuboid(cubeName, segmentId, metaUrl));
Long totalCount = 0L;
if (envConfig.isSparkSanityCheckEnabled()) {
totalCount = encodedBaseRDD.count();
}
-final BaseCuboidReducerFunction2 baseCuboidReducerFunction = new
BaseCuboidReducerFunction2(cubeName);
+final BaseCuboidReducerFunction2 baseCuboidReducerFunction = new
BaseCuboidReducerFunction2(cubeName, metaUrl);
BaseCuboidReducerFunction2 reducerFunction2 =
baseCuboidReducerFunction;
if (allNormalMeasure == false) {
-reducerFunction2 = new CuboidReducerFunction2(cubeName, needAggr);
+reducerFunction2 = new CuboidReducerFunction2(cubeName, metaUrl,
needAggr);
}
final int totalLevels = cubeDesc.getBuildLevel();
@@ -202,18 +201,18 @@ public class SparkCubingByLayer extends
AbstractApplication implements Serializa
Configuration confOverwrite = new
Configuration(sc.hadoopConfiguration());
confOverwrite.set("dfs.replication", "2"); // cuboid intermediate
files, replication=2
-saveToHDFS(allRDDs[0], cubeName, cubeSegment, outputPath, 0,
confOverwrite);
+saveToHDFS(allRDDs[0], cubeName, metaUrl, cubeSegment, outputPath, 0,
confOverwrite);
// aggregate to ND cuboids
for (level = 1; level <= totalLevels; level++) {
long levelRddSize = SizeEstimator.estimate(allRDDs[level - 1]) /
(1024 * 1024);
partition = estimateRDDPartitionNum(level, cubeStatsReader,
envConfig, (int) levelRddSize);
-allRDDs[level] = allRDDs[level - 1].flatMapToPair(new
CuboidFlatMap(cubeName, segmentId))
+allRDDs[level] = allRDDs[level - 1].flatMapToPair(new
CuboidFlatMap(cubeName, segmentId, metaUrl))
.reduceByKey(reducerFunction2,
partition).persist(storageLevel);
if
[23/50] [abbrv] kylin git commit: Revert "KYLIN-2816 support hive unicode column name"
Revert "KYLIN-2816 support hive unicode column name"
This reverts commit 3712464e5e09c66b4c17cdb0c19e420c61c13358.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/487a854b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/487a854b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/487a854b
Branch: refs/heads/2622-2764
Commit: 487a854bfece7b9fc72b0118a40d51100eca8463
Parents: 81e9f57
Author: Hongbin Ma
Authored: Mon Aug 28 19:19:24 2017 +0800
Committer: Hongbin Ma
Committed: Tue Aug 29 10:36:10 2017 +0800
--
.../kylin/common/util/HiveCmdBuilder.java | 5 ++--
.../kylin/common/util/HiveCmdBuilderTest.java | 2 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 28
.../apache/kylin/source/hive/HiveMRInput.java | 2 +-
4 files changed, 9 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 191bbc0..2f6b9a0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -66,12 +66,11 @@ public class HiveCmdBuilder {
switch (clientMode) {
case CLI:
-// use single-quote to ignore the executing back-ticks surrounded
column name in shell
-buf.append("hive -e \'");
+buf.append("hive -e \"");
for (String statement : statements) {
buf.append(statement).append("\n");
}
-buf.append("\'");
+buf.append("\"");
buf.append(parseProps());
break;
case BEELINE:
http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index d2938b5..d69d4d2 100644
---
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -60,7 +60,7 @@ public class HiveCmdBuilderTest {
hiveCmdBuilder.addStatement("SHOW\n TABLES;");
hiveCmdBuilder.setHiveConfProps(hiveProps);
hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite);
-assertEquals("hive -e \'USE default;\nDROP TABLE test;\nSHOW\n
TABLES;\n\' --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
+assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n
TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 824c693..acb29e1 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -42,8 +42,6 @@ import org.apache.kylin.metadata.model.TblColRef;
public class JoinedFlatTable {
-public static final String BACKTICK = "`";
-
public static String getTableDir(IJoinedFlatTableDesc flatDesc, String
storageDfsDir) {
return storageDfsDir + "/" + flatDesc.getTableName();
}
@@ -77,7 +75,7 @@ public class JoinedFlatTable {
ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
}
ddl.append("STORED AS " + format + "\n");
-ddl.append("LOCATION '\"'\"'" + getTableDir(flatDesc, storageDfsDir) +
"'\"'\"';").append("\n");
+ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) +
"';").append("\n");
return ddl.toString();
}
@@ -130,15 +128,10 @@ public class JoinedFlatTable {
sql.append(",");
}
String colTotalName = String.format("%s.%s",
col.getTableRef().getTableName(), col.getName());
-String expressionInSourceDB = col.getExpressionInSourceDB();
-if (expressionInSourceDB.contains(".")) {
-// surround column name with back-tick, to support unicode
column name
-expre
kylin git commit: KYLIN-2838 Should get storageType in changeHtableHost of CubeMigrationCLI
Repository: kylin
Updated Branches:
refs/heads/master 34596d355 -> 78543d6e9
KYLIN-2838 Should get storageType in changeHtableHost of CubeMigrationCLI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78543d6e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78543d6e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78543d6e
Branch: refs/heads/master
Commit: 78543d6e970cfb9dc85bcc48775681afcdb1c0e9
Parents: 34596d3
Author: kangkaisen
Authored: Sun Sep 3 17:52:18 2017 +0800
Committer: kangkaisen
Committed: Sun Sep 3 17:53:37 2017 +0800
--
tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/78543d6e/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index bb308e2..d3865c6 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -201,7 +201,7 @@ public class CubeMigrationCLI {
}
protected void changeHtableHost(CubeInstance cube) {
-if (cube.getDescriptor().getEngineType() !=
IStorageAware.ID_SHARDED_HBASE)
+if (cube.getDescriptor().getStorageType() !=
IStorageAware.ID_SHARDED_HBASE)
return;
for (CubeSegment segment : cube.getSegments()) {
operations
[1/2] kylin git commit: KYLIN-2622 Add SegmentAppendTrieDictBuilder for count distinct measure
Repository: kylin
Updated Branches:
refs/heads/2622-2764 [created] 2607e18b5
KYLIN-2622 Add SegmentAppendTrieDictBuilder for count distinct measure
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ec5dd54e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ec5dd54e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ec5dd54e
Branch: refs/heads/2622-2764
Commit: ec5dd54e9ea5e373569cd65cab322a17716718ff
Parents: 15d6063
Author: kangkaisen
Authored: Tue May 16 17:04:55 2017 +0800
Committer: kangkaisen
Committed: Sat Sep 2 18:53:07 2017 +0800
--
.../apache/kylin/dict/AppendTrieDictionary.java | 9 ++-
.../apache/kylin/dict/DictionaryManager.java| 10 +++
.../kylin/dict/GlobalDictionaryBuilder.java | 8 +--
.../global/AppendTrieDictionaryBuilder.java | 15 ++--
.../kylin/dict/global/GlobalDictHDFSStore.java | 54 +-
.../kylin/dict/global/GlobalDictMetadata.java | 1 -
.../kylin/dict/global/GlobalDictStore.java | 5 +-
.../global/SegmentAppendTrieDictBuilder.java| 76
.../dict/global/AppendTrieDictionaryTest.java | 3 +-
.../kylin/measure/bitmap/BitmapMeasureType.java | 21 +-
.../localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
webapp/app/js/model/cubeConfig.js | 3 +-
.../cubeDesigner/advanced_settings.html | 13 +++-
13 files changed, 162 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ec5dd54e/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index b9f0d2b..18795b4 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -38,8 +38,8 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.Objects;
+import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
-import static com.google.common.base.Preconditions.checkState;
/**
* A dictionary based on Trie data structure that maps enumerations of byte[]
to
@@ -72,7 +72,12 @@ public class AppendTrieDictionary extends
CacheDictionary {
this.baseDir = baseDir;
final GlobalDictStore globalDictStore = new
GlobalDictHDFSStore(baseDir);
Long[] versions = globalDictStore.listAllVersions();
-checkState(versions.length > 0, "Global dict at %s is empty", baseDir);
+
+if (versions.length == 0) {
+this.metadata = new GlobalDictMetadata(0, 0, 0, 0, null, new
TreeMap());
+return; // for the removed SegmentAppendTrieDictBuilder
+}
+
final long latestVersion = versions[versions.length - 1];
final Path latestVersionPath =
globalDictStore.getVersionDir(latestVersion);
this.metadata = globalDictStore.getMetadata(latestVersion);
http://git-wip-us.apache.org/repos/asf/kylin/blob/ec5dd54e/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 857ee30..47e5c29 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -227,6 +227,16 @@ public class DictionaryManager {
if (dicts.size() == 1)
return dicts.get(0);
+/**
+ * AppendTrieDictionary needn't merge
+ * more than one AppendTrieDictionary will generate when user use
{@link SegmentAppendTrieDictBuilder}
+ */
+for (DictionaryInfo dict: dicts) {
+if
(dict.getDictionaryClass().equals(AppendTrieDictionary.class.getName())) {
+return dict;
+}
+}
+
DictionaryInfo firstDictInfo = null;
int totalSize = 0;
for (DictionaryInfo info : dicts) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ec5dd54e/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
index a593371..404d53c 100644
---
a/core-dictionary/src/main/java/org
[2/2] kylin git commit: KYLIN-2764 Build UHC Dict Use MR
KYLIN-2764 Build UHC Dict Use MR
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2607e18b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2607e18b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2607e18b
Branch: refs/heads/2622-2764
Commit: 2607e18b5e17d2a68f4079a76b8c990f144cbbd6
Parents: ec5dd54
Author: kangkaisen
Authored: Fri Jul 21 15:22:30 2017 +0800
Committer: kangkaisen
Committed: Sat Sep 2 18:53:17 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 11 +-
.../java/org/apache/kylin/cube/CubeManager.java | 82 +-
.../apache/kylin/dict/DictionaryGenerator.java | 14 +-
.../org/apache/kylin/dict/DictionaryInfo.java | 4 +
.../kylin/dict/GlobalDictionaryBuilder.java | 9 +-
.../apache/kylin/dict/IDictionaryBuilder.java | 2 +-
.../global/SegmentAppendTrieDictBuilder.java| 4 +-
.../kylin/dict/DictionaryProviderTest.java | 2 +-
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 26 +++-
.../kylin/engine/mr/JobBuilderSupport.java | 24 ++-
.../engine/mr/common/AbstractHadoopJob.java | 1 +
.../kylin/engine/mr/common/BatchConstants.java | 4 +
.../engine/mr/steps/CreateDictionaryJob.java| 16 +-
.../mr/steps/FactDistinctColumnsReducer.java| 2 +-
.../kylin/engine/mr/steps/UHCDictionaryJob.java | 154 +++
.../engine/mr/steps/UHCDictionaryMapper.java| 101
.../mr/steps/UHCDictionaryPartitioner.java | 30
.../engine/mr/steps/UHCDictionaryReducer.java | 113 ++
.../test_case_data/sandbox/kylin.properties | 5 +
.../dict/ITGlobalDictionaryBuilderTest.java | 4 +-
21 files changed, 551 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/2607e18b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f9c3adb..16c60de 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -844,6 +844,10 @@ abstract public class KylinConfigBase implements
Serializable {
return getPropertiesByPrefix("kylin.engine.mr.config-override.");
}
+public Map getUHCMRConfigOverride() {
+return getPropertiesByPrefix("kylin.engine.mr.uhc-config-override.");
+}
+
public Map getSparkConfigOverride() {
return getPropertiesByPrefix("kylin.engine.spark-conf.");
}
@@ -870,9 +874,14 @@ abstract public class KylinConfigBase implements
Serializable {
//UHC: ultra high cardinality columns, contain the ShardByColumns and the
GlobalDictionaryColumns
public int getUHCReducerCount() {
-return
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
+return
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "5"));
}
+public boolean isBuildUHCDictWithMREnabled() {
+return
Boolean.parseBoolean(getOptional("kylin.engine.mr.build-uhc-dict", "true"));
+}
+
+
public boolean isBuildDictInReducerEnabled() {
return
Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer",
"true"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2607e18b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 0beb926..aaaedd3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -30,7 +30,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -227,6 +226,7 @@ public class CubeManager implements IRealizationProvider {
return null;
String builderClass = cubeDesc.getDictionaryBuilderClass(col);
+
DictionaryInfo dictInfo =
getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable,
builderClass);
@@ -907,6 +907,52 @@ public class CubeManager implements IRealizationProvider {
return factDictCols;
kylin git commit: KYLIN-2606 Only return counter for precise count_distinct if query is exactAggregate
Repository: kylin
Updated Branches:
refs/heads/master ef148601d -> 5b942267f
KYLIN-2606 Only return counter for precise count_distinct if query is
exactAggregate
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5b942267
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5b942267
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5b942267
Branch: refs/heads/master
Commit: 5b942267fc26175199b4b26229e54def6728ce87
Parents: ef14860
Author: kangkaisen
Authored: Wed Feb 15 19:53:17 2017 +0800
Committer: kangkaisen
Committed: Sat Aug 19 13:30:36 2017 +0800
--
.../kylin/cube/gridtable/CubeCodeSystem.java| 4 +
.../org/apache/kylin/gridtable/GTRecord.java| 8 +
.../kylin/gridtable/GTSampleCodeSystem.java | 4 +
.../apache/kylin/gridtable/IGTCodeSystem.java | 3 +
.../measure/bitmap/BitmapCounterFactory.java| 2 +
.../kylin/measure/bitmap/BitmapSerializer.java | 38 -
.../measure/bitmap/RoaringBitmapCounter.java| 10 ++
.../bitmap/RoaringBitmapCounterFactory.java | 5 +
.../metadata/datatype/DataTypeSerializer.java | 13 ++
.../gtrecord/GTCubeStorageQueryBase.java| 53 ++-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 1 +
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 2 +-
.../hbase/cube/v2/HBaseReadonlyStore.java | 33 +++-
.../coprocessor/endpoint/CubeVisitService.java | 2 +-
.../endpoint/generated/CubeVisitProtos.java | 151 +++
.../endpoint/protobuf/CubeVisit.proto | 1 +
16 files changed, 292 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/5b942267/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index aaa12a7..9eae6f3 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -177,4 +177,8 @@ public class CubeCodeSystem implements IGTCodeSystem {
return result;
}
+@Override
+public DataTypeSerializer getSerializer(int col) {
+return serializers[col];
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5b942267/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 3e62ea7..f65e4b5 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -296,4 +296,12 @@ public class GTRecord implements Comparable,
Cloneable {
}
}
+/** change pointers to point to data in given buffer, this
+ * method allows to defined specific column to load */
+public void loadColumns(int selectedCol, ByteBuffer buf) {
+int pos = buf.position();
+int len = info.codeSystem.codeLength(selectedCol, buf);
+cols[selectedCol].set(buf.array(), buf.arrayOffset() + pos, len);
+}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5b942267/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index 3f3c844..2a5e791 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -118,4 +118,8 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
}
};
+@Override
+public DataTypeSerializer getSerializer(int col) {
+return serializers[col];
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5b942267/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
index 89dfc99..9c8ad6b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.ImmutableBit
kylin git commit: KYLIN-2706 Only compare with group columns in SortedIteratorMergerWithLimit
Repository: kylin
Updated Branches:
refs/heads/master 714b22512 -> 659eeaedd
KYLIN-2706 Only compare with group columns in SortedIteratorMergerWithLimit
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/659eeaed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/659eeaed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/659eeaed
Branch: refs/heads/master
Commit: 659eeaedd571c837df3beae44456dadde3036c3d
Parents: 714b225
Author: kangkaisen
Authored: Sat Jul 22 20:40:45 2017 +0800
Committer: kangkaisen
Committed: Wed Aug 9 19:59:58 2017 +0800
--
.../gtrecord/GTCubeStorageQueryBase.java| 2 +-
.../gtrecord/SequentialCubeTupleIterator.java | 17
.../test/resources/query/sql_limit/query05.sql | 21
3 files changed, 35 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/659eeaed/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index f89fc47..3543c75 100644
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -94,7 +94,7 @@ public abstract class GTCubeStorageQueryBase implements
IStorageQuery {
return ITupleIterator.EMPTY_TUPLE_ITERATOR;
return new SequentialCubeTupleIterator(scanners, request.getCuboid(),
request.getDimensions(),
-request.getMetrics(), returnTupleInfo, request.getContext(),
sqlDigest);
+request.getGroups(), request.getMetrics(), returnTupleInfo,
request.getContext(), sqlDigest);
}
protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext
context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/659eeaed/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
--
diff --git
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index f7cb1b9..3cbb538 100644
---
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -23,6 +23,7 @@ import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import java.util.TreeSet;
import org.apache.kylin.common.exceptions.KylinTimeoutException;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -53,7 +54,7 @@ public class SequentialCubeTupleIterator implements
ITupleIterator {
private int scanCountDelta;
public SequentialCubeTupleIterator(List scanners,
Cuboid cuboid, Set selectedDimensions, //
-Set selectedMetrics, TupleInfo returnTupleInfo,
StorageContext context, SQLDigest sqlDigest) {
+Set groups, Set selectedMetrics,
TupleInfo returnTupleInfo, StorageContext context, SQLDigest sqlDigest) {
this.context = context;
this.scanners = scanners;
@@ -66,7 +67,7 @@ public class SequentialCubeTupleIterator implements
ITupleIterator {
//query with limit
logger.info("Using SortedIteratorMergerWithLimit to merge segment
results");
Iterator> transformed =
(Iterator>) (Iterator) segmentCubeTupleIterators.iterator();
-tupleIterator = new
SortedIteratorMergerWithLimit(transformed,
context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid,
returnTupleInfo)).getIterator();
+tupleIterator = new
SortedIteratorMergerWithLimit(transformed,
context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, groups,
returnTupleInfo)).getIterator();
} else {
//normal case
logger.info("Using Iterators.concat to merge segment results");
@@ -74,10 +75,18 @@ public class SequentialCubeTupleIterator implements
ITupleIterator {
}
}
-public Comparator getTupleDimensionComparator(Cuboid cuboid,
TupleInfo returnTupleInfo) {
+public Comparator getTupleDimensionComparator(Cuboid cuboid,
Set groups, TupleInfo returnTupleInfo) {
// dimensionIndexOnTuple is for SQL with limit
+List dimColumns = cuboid.getColumns();
+
+TreeSet
kylin git commit: Revert "KYLIN-2604 Use global dict as the default encoding for precise distinct count in web"
Repository: kylin
Updated Branches:
refs/heads/v2.1.0-release a85a72db3 -> 74912d48d
Revert "KYLIN-2604 Use global dict as the default encoding for precise distinct
count in web"
This reverts commit 675881f5326224ca6723499604d8391ce894f05b.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74912d48
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74912d48
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74912d48
Branch: refs/heads/v2.1.0-release
Commit: 74912d48d6d6fe7507f5b84f4010f957fabcaef2
Parents: a85a72d
Author: kangkaisen
Authored: Wed Aug 9 19:41:12 2017 +0800
Committer: kangkaisen
Committed: Wed Aug 9 19:41:12 2017 +0800
--
webapp/app/js/controllers/cubeAdvanceSetting.js | 5 ++
webapp/app/js/controllers/cubeEdit.js | 52
webapp/app/js/controllers/cubeMeasures.js | 7 ---
3 files changed, 5 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/74912d48/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 8041a12..7d375da 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -187,6 +187,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function
($scope, $modal,cubeConfi
};
$scope.initUpdateDictionariesStatus();
+
$scope.addNewDictionaries = function (dictionaries, index) {
if(dictionaries&&index>=0){
$scope.updateDictionariesStatus.isEdit = true;
@@ -206,6 +207,10 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function
($scope, $modal,cubeConfi
};
$scope.saveNewDictionaries = function (){
+if(!$scope.cubeMetaFrame.dictionaries){
+ $scope.cubeMetaFrame.dictionaries=[];
+}
+
if($scope.updateDictionariesStatus.isEdit == true) {
if
($scope.cubeMetaFrame.dictionaries[$scope.updateDictionariesStatus.editIndex].column
!= $scope.newDictionaries.column) {
if(!$scope.checkColumn()){
http://git-wip-us.apache.org/repos/asf/kylin/blob/74912d48/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js
b/webapp/app/js/controllers/cubeEdit.js
index 5f78b61..0e5038e 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -721,59 +721,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q,
$routeParams, $locatio
}
}
- function reGenerateAdvancedDict () {
-if (!$scope.cubeMetaFrame.dictionaries) {
- $scope.cubeMetaFrame.dictionaries = [];
-}
-
-var distinctMeasures = [];
-
-angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
- if (measure.function.expression === 'COUNT_DISTINCT' &&
measure.function.returntype === 'bitmap') {
-var measureColumn = measure.function.parameter.value;
-distinctMeasures.push(measureColumn);
-//keep backward compatibility
-distinctMeasures.push(VdmUtil.removeNameSpace(measureColumn))
-
-var isColumnExit = false;
-angular.forEach($scope.cubeMetaFrame.dictionaries, function
(dictionaries) {
- if (!isColumnExit) {
-//keep backward compatibility
-if (dictionaries.column == measureColumn || dictionaries.column ==
VdmUtil.removeNameSpace(measureColumn))
- isColumnExit = true;
- }
-});
-
-if (!isColumnExit) {
- var dict = CubeDescModel.createDictionaries();
- dict.column = measureColumn;
- dict.builder = cubeConfig.buildDictionaries[0].value;
- $scope.cubeMetaFrame.dictionaries.push(dict)
-}
- }
-});
-
-//get all reuse columns
-var reuseColumns = [];
-angular.forEach($scope.cubeMetaFrame.dictionaries, function (dict, index) {
- if (dict.reuse != null && reuseColumns.indexOf(dict.reuse) === -1) {
-reuseColumns.push(dict.reuse);
- }
-});
-//remove deprecated distinct measures
-angular.forEach($scope.cubeMetaFrame.dictionaries, function (dict, index) {
- if (distinctMeasures.indexOf(dict.column) === -1 &&
reuseColumns.indexOf(dict.column) === -1) {
-$scope.cubeMetaFrame.dictionaries.splice(index, 1);
- }
-});
- }
-
- $scope.$on('MeasuresEdited', function (event) {
-if ($scope.cubeMetaFrame) {
- reGenerateAdvancedDict();
-}
- });
$scope.$on('DimensionsEdited', function (event) {
if ($scope.
kylin git commit: KryoRegistrator add ComputedColumnDesc
Repository: kylin
Updated Branches:
refs/heads/master c08965f92 -> 03bea8d94
KryoRegistrator add ComputedColumnDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03bea8d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03bea8d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03bea8d9
Branch: refs/heads/master
Commit: 03bea8d94c2d4e948c45905cdc174fde73648cd8
Parents: c08965f
Author: kangkaisen
Authored: Thu Jul 13 20:18:17 2017 +0800
Committer: kangkaisen
Committed: Thu Jul 27 20:50:10 2017 +0800
--
.../java/org/apache/kylin/metadata/model/ComputedColumnDesc.java | 4 +++-
.../java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java | 1 +
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/03bea8d9/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 540b5fc..e8cc351 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -22,8 +22,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.apache.kylin.metadata.model.tool.CalciteParser;
+import java.io.Serializable;
+
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE,
getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility =
JsonAutoDetect.Visibility.NONE, setterVisibility =
JsonAutoDetect.Visibility.NONE)
-public class ComputedColumnDesc {
+public class ComputedColumnDesc implements Serializable {
@JsonProperty
private String tableIdentity;
@JsonProperty
http://git-wip-us.apache.org/repos/asf/kylin/blob/03bea8d9/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
--
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index 106653f..048f62b 100644
---
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -269,6 +269,7 @@ public class KylinKryoRegistrator implements
KryoRegistrator {
kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class);
kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class);
kyroClasses.add(org.apache.kylin.metadata.model.DatabaseDesc.class);
+
kyroClasses.add(org.apache.kylin.metadata.model.ComputedColumnDesc.class);
kyroClasses.add(org.apache.kylin.metadata.model.ExternalFilterDesc.class);
kyroClasses.add(org.apache.kylin.metadata.model.FunctionDesc.class);
kyroClasses.add(org.apache.kylin.metadata.model.JoinDesc.class);
kylin git commit: KYLIN-2653 Manage the metadata use HDFSResourceStore for Spark Cubing
Repository: kylin
Updated Branches:
refs/heads/master 6f53bdc54 -> c08965f92
KYLIN-2653 Manage the metadata use HDFSResourceStore for Spark Cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c08965f9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c08965f9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c08965f9
Branch: refs/heads/master
Commit: c08965f92906bdd6f38efdbe92cbdace570681c7
Parents: 6f53bdc
Author: kangkaisen
Authored: Thu May 25 14:43:02 2017 +0800
Committer: kangkaisen
Committed: Thu Jul 27 20:46:48 2017 +0800
--
.../org/apache/kylin/common/KylinConfig.java| 35 -
.../org/apache/kylin/common/KylinConfigExt.java | 2 +-
.../engine/mr/common/AbstractHadoopJob.java | 45 +--
.../engine/mr/common/JobRelatedMetaUtil.java| 71 +
.../spark/SparkBatchCubingJobBuilder2.java | 6 +-
.../kylin/engine/spark/SparkCubingByLayer.java | 82 ++--
.../kylin/engine/spark/SparkExecutable.java | 58 +++---
7 files changed, 188 insertions(+), 111 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c08965f9/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index cc08056..a56e9b8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -36,8 +36,11 @@ import java.util.Properties;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.OrderedProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -114,10 +117,13 @@ public class KylinConfig extends KylinConfigBase {
}
public enum UriType {
-PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER
+PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER, HDFS_FILE
}
private static UriType decideUriType(String metaUri) {
+if (metaUri.indexOf("@hdfs") > 0) {
+return UriType.HDFS_FILE;
+}
try {
File file = new File(metaUri);
@@ -157,6 +163,23 @@ public class KylinConfig extends KylinConfigBase {
*/
UriType uriType = decideUriType(uri);
+if (uriType == UriType.HDFS_FILE) {
+KylinConfig config;
+FileSystem fs;
+int cut = uri.indexOf('@');
+String realHdfsPath = uri.substring(0, cut) + "/" +
KYLIN_CONF_PROPERTIES_FILE;
+try {
+config = new KylinConfig();
+fs = HadoopUtil.getFileSystem(realHdfsPath);
+InputStream is = fs.open(new Path(realHdfsPath));
+Properties prop = streamToProps(is);
+config.reloadKylinConfig(prop);
+} catch (IOException e) {
+throw new RuntimeException(e);
+}
+return config;
+}
+
if (uriType == UriType.LOCAL_FOLDER) {
KylinConfig config = new KylinConfig();
config.setMetadataUrl(uri);
@@ -402,6 +425,16 @@ public class KylinConfig extends KylinConfigBase {
super(props, force);
}
+public void writeProperties(Properties props, File file) throws
IOException {
+FileOutputStream fos = null;
+try {
+fos = new FileOutputStream(file);
+props.store(fos, file.getAbsolutePath());
+} finally {
+IOUtils.closeQuietly(fos);
+}
+}
+
public void writeProperties(File file) throws IOException {
FileOutputStream fos = null;
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c08965f9/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
index d49dee7..786f467 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
@@ -61,7 +61,7 @@ public class KylinConfigExt extends KylinConfig {
return super.getOptional(prop, dft);
}
-protected Properties getAllProperties() {
+
[1/4] kylin git commit: KYLIN-2653 Manage the metadata use HDFSResourceStore for Spark Cubing [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2653 edbe7a52f -> 276fa2c4d (forced update)
KYLIN-2653 Manage the metadata use HDFSResourceStore for Spark Cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d8d0395a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d8d0395a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d8d0395a
Branch: refs/heads/KYLIN-2653
Commit: d8d0395a80cc50fcb59bab4d402c7675aef6cd22
Parents: 1a3527c
Author: kangkaisen
Authored: Thu May 25 14:43:02 2017 +0800
Committer: kangkaisen
Committed: Sat Jul 22 14:59:47 2017 +0800
--
.../org/apache/kylin/common/KylinConfig.java| 35 -
.../org/apache/kylin/common/KylinConfigExt.java | 2 +-
.../engine/mr/common/AbstractHadoopJob.java | 45 +--
.../engine/mr/common/JobRelatedMetaUtil.java| 71 +
.../spark/SparkBatchCubingJobBuilder2.java | 6 +-
.../kylin/engine/spark/SparkCubingByLayer.java | 82 ++--
.../kylin/engine/spark/SparkExecutable.java | 58 +++---
7 files changed, 188 insertions(+), 111 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d8d0395a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index cc08056..a56e9b8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -36,8 +36,11 @@ import java.util.Properties;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.OrderedProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -114,10 +117,13 @@ public class KylinConfig extends KylinConfigBase {
}
public enum UriType {
-PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER
+PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER, HDFS_FILE
}
private static UriType decideUriType(String metaUri) {
+if (metaUri.indexOf("@hdfs") > 0) {
+return UriType.HDFS_FILE;
+}
try {
File file = new File(metaUri);
@@ -157,6 +163,23 @@ public class KylinConfig extends KylinConfigBase {
*/
UriType uriType = decideUriType(uri);
+if (uriType == UriType.HDFS_FILE) {
+KylinConfig config;
+FileSystem fs;
+int cut = uri.indexOf('@');
+String realHdfsPath = uri.substring(0, cut) + "/" +
KYLIN_CONF_PROPERTIES_FILE;
+try {
+config = new KylinConfig();
+fs = HadoopUtil.getFileSystem(realHdfsPath);
+InputStream is = fs.open(new Path(realHdfsPath));
+Properties prop = streamToProps(is);
+config.reloadKylinConfig(prop);
+} catch (IOException e) {
+throw new RuntimeException(e);
+}
+return config;
+}
+
if (uriType == UriType.LOCAL_FOLDER) {
KylinConfig config = new KylinConfig();
config.setMetadataUrl(uri);
@@ -402,6 +425,16 @@ public class KylinConfig extends KylinConfigBase {
super(props, force);
}
+public void writeProperties(Properties props, File file) throws
IOException {
+FileOutputStream fos = null;
+try {
+fos = new FileOutputStream(file);
+props.store(fos, file.getAbsolutePath());
+} finally {
+IOUtils.closeQuietly(fos);
+}
+}
+
public void writeProperties(File file) throws IOException {
FileOutputStream fos = null;
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d8d0395a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
index d49dee7..786f467 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
@@ -61,7 +61,7 @@ public class KylinConfigExt extends KylinConfig {
return super.getOptional(prop, dft);
}
-protected Propert
[3/4] kylin git commit: Tmp annotated CalciteParser.ensureNoTableNameExists
Tmp annotated CalciteParser.ensureNoTableNameExists
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/276fa2c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/276fa2c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/276fa2c4
Branch: refs/heads/KYLIN-2653
Commit: 276fa2c4dafd22cd897227d5cc592cd528f40cc5
Parents: 3326b08
Author: kangkaisen
Authored: Fri Jul 14 21:31:49 2017 +0800
Committer: kangkaisen
Committed: Sat Jul 22 14:59:54 2017 +0800
--
.../java/org/apache/kylin/metadata/model/ComputedColumnDesc.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/276fa2c4/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index e8cc351..13cb889 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -20,7 +20,6 @@ package org.apache.kylin.metadata.model;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
import java.io.Serializable;
@@ -46,7 +45,7 @@ public class ComputedColumnDesc implements Serializable {
tableIdentity = tableIdentity.toUpperCase();
columnName = columnName.toUpperCase();
-CalciteParser.ensureNoTableNameExists(expression);
+//CalciteParser.ensureNoTableNameExists(expression);
}
public String getFullName() {
[2/4] kylin git commit: build ci_inner_join_cube with spark
build ci_inner_join_cube with spark
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6413c286
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6413c286
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6413c286
Branch: refs/heads/KYLIN-2653
Commit: 6413c286ea1d18ea1a8bb2bf9e975e162a08b39f
Parents: d8d0395
Author: kangkaisen
Authored: Thu Jul 13 16:46:41 2017 +0800
Committer: kangkaisen
Committed: Sat Jul 22 14:59:54 2017 +0800
--
.../test_case_data/localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/6413c286/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
--
diff --git
a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index 28a63d5..27acdd3 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -610,7 +610,7 @@
"status_need_notify": [],
"auto_merge_time_ranges": null,
"retention_range": 0,
- "engine_type": 2,
+ "engine_type": 4,
"storage_type": 2,
"override_kylin_properties": {
"kylin.cube.algorithm": "LAYER"
