date:20151222

svn commit: r11725 - in /dev/kylin/apache-kylin-1.3-snapshot: ./ apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Wed Dec 23 02:48:47 2015
New Revision: 11725

Log:
add apache-kylin-1.3-snapshot for hbase1.1

Added:
dev/kylin/apache-kylin-1.3-snapshot/

dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
   (with props)

Added: 
dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
==
Binary file - no diff available.

Propchange: 
dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
--
svn:mime-type = application/octet-stream

[05/50] [abbrv] kylin git commit: KYLIN-1160 code review

2015-12-22 Thread liyang

KYLIN-1160 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55c448b3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55c448b3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55c448b3

Branch: refs/heads/1.x-HBase1.1.3
Commit: 55c448b30907eefca3561b8c15c61baccc4fba66
Parents: b7b6793
Author: Li, Yang 
Authored: Wed Nov 25 15:39:39 2015 +0800
Committer: Li, Yang 
Committed: Wed Nov 25 15:39:39 2015 +0800

--
 .../main/java/org/apache/kylin/common/util/Log4jConfigurer.java  | 4 ++--
 .../main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java| 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/55c448b3/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java 
b/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
index c329ced..6f887b7 100644
--- a/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
+++ b/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
@@ -17,15 +17,15 @@ public class Log4jConfigurer {
 public static void initLogger() {
 if (!INITIALIZED && !isConfigured()) {
 org.apache.log4j.BasicConfigurator.configure(new 
ConsoleAppender(new PatternLayout(DEFAULT_PATTERN_LAYOUT)));
-INITIALIZED = true;
 }
+INITIALIZED = true;
 }
 
 private static boolean isConfigured() {
 if (LogManager.getRootLogger().getAllAppenders().hasMoreElements()) {
 return true;
 } else {
-Enumeration loggers = LogManager.getCurrentLoggers();
+Enumeration loggers = LogManager.getCurrentLoggers();
 while (loggers.hasMoreElements()) {
 Logger logger = (Logger) loggers.nextElement();
 if (logger.getAllAppenders().hasMoreElements())

http://git-wip-us.apache.org/repos/asf/kylin/blob/55c448b3/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
index 715d190..8bf35e2 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/util/Log4jConfigurer.java
@@ -17,15 +17,15 @@ public class Log4jConfigurer {
 public static void initLogger() {
 if (!INITIALIZED && !isConfigured()) {
 org.apache.log4j.BasicConfigurator.configure(new 
ConsoleAppender(new PatternLayout(DEFAULT_PATTERN_LAYOUT)));
-INITIALIZED = true;
 }
+INITIALIZED = true;
 }
 
 private static boolean isConfigured() {
 if (LogManager.getRootLogger().getAllAppenders().hasMoreElements()) {
 return true;
 } else {
-Enumeration loggers = LogManager.getCurrentLoggers();
+Enumeration loggers = LogManager.getCurrentLoggers();
 while (loggers.hasMoreElements()) {
 Logger logger = (Logger) loggers.nextElement();
 if (logger.getAllAppenders().hasMoreElements())

[37/50] [abbrv] kylin git commit: KYLIN-976 Support Custom Aggregation Types

2015-12-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/c721d679/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index fd86138..f225fe2 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -59,7 +59,6 @@ import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.query.sqlfunc.HLLDistinctCountAggFunc;
 
 import com.google.common.base.Preconditions;
 
@@ -74,7 +73,6 @@ public class OLAPAggregateRel extends Aggregate implements 
OLAPRel {
 AGGR_FUNC_MAP.put("$SUM0", "SUM");
 AGGR_FUNC_MAP.put("COUNT", "COUNT");
 AGGR_FUNC_MAP.put("COUNT_DISTINCT", "COUNT_DISTINCT");
-AGGR_FUNC_MAP.put("HLL_COUNT", "COUNT_DISTINCT");
 AGGR_FUNC_MAP.put("MAX", "MAX");
 AGGR_FUNC_MAP.put("MIN", "MIN");
 }
@@ -301,10 +299,10 @@ public class OLAPAggregateRel extends Aggregate 
implements OLAPRel {
 // rebuild function
 RelDataType fieldType = aggCall.getType();
 SqlAggFunction newAgg = aggCall.getAggregation();
-if (func.isCountDistinct()) {
-newAgg = createHyperLogLogAggFunction(fieldType);
-} else if (func.isCount()) {
+if (func.isCount()) {
 newAgg = SqlStdOperatorTable.SUM0;
+} else if (func.getMeasureType().getRewriteCalciteAggrFunctionClass() 
!= null) {
+newAgg = createCustomAggFunction(func.getExpression(), fieldType, 
func.getMeasureType().getRewriteCalciteAggrFunctionClass());
 }
 
 // rebuild aggregate call
@@ -312,10 +310,10 @@ public class OLAPAggregateRel extends Aggregate 
implements OLAPRel {
 return newAggCall;
 }
 
-private SqlAggFunction createHyperLogLogAggFunction(RelDataType 
returnType) {
+private SqlAggFunction createCustomAggFunction(String funcName, 
RelDataType returnType, Class customAggFuncClz) {
 RelDataTypeFactory typeFactory = getCluster().getTypeFactory();
-SqlIdentifier sqlIdentifier = new SqlIdentifier("HLL_COUNT", new 
SqlParserPos(1, 1));
-AggregateFunction aggFunction = 
AggregateFunctionImpl.create(HLLDistinctCountAggFunc.class);
+SqlIdentifier sqlIdentifier = new SqlIdentifier(funcName, new 
SqlParserPos(1, 1));
+AggregateFunction aggFunction = 
AggregateFunctionImpl.create(customAggFuncClz);
 List argTypes = new ArrayList();
 List typeFamilies = new ArrayList();
 for (FunctionParameter o : aggFunction.getParameters()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c721d679/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java 
b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
new file mode 100644
index 000..22608a0
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.routing;
+
+import java.util.Map;
+
+import org.apache.kylin.metadata.realization.CapabilityResult;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+
+import com.google.common.collect.Maps;
+
+public class Candidate implements Comparable {
+
+static final Map PRIORITIES = Maps.newHashMap();
+
+static {
+PRIORITIES.put(RealizationType.HYBRID, 0);
+PRIORITIES.put(RealizationType.CUBE, 0);
+PRIORITIES.put(RealizationType.INVERTED_INDEX,

[42/50] [abbrv] kylin git commit: KYLIN-976 remove holistic hll in cube desc

2015-12-22 Thread liyang

KYLIN-976 remove holistic hll in cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/623e70f4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/623e70f4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/623e70f4

Branch: refs/heads/1.x-HBase1.1.3
Commit: 623e70f47624dd67de01b02cf8b227cf5381eae5
Parents: c721d67
Author: honma 
Authored: Tue Dec 15 18:23:21 2015 +0800
Committer: honma 
Committed: Tue Dec 15 18:23:21 2015 +0800

--
 .../cube_desc/test_kylin_cube_without_slr_desc.json | 16 +---
 .../test_kylin_cube_without_slr_left_join_desc.json | 16 +---
 2 files changed, 2 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/623e70f4/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
index fedb2c1..bf29268 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json
@@ -151,19 +151,6 @@
 },
 {
   "id": 7,
-  "name": "SELLER_CNT_LONG",
-  "function": {
-"expression": "COUNT_DISTINCT",
-"parameter": {
-  "type": "column",
-  "value": "SELLER_ID"
-},
-"returntype": "bigint"
-  },
-  "dependent_measure_ref": "SELLER_CNT_HLL"
-},
-{
-  "id": 8,
   "name": "SELLER_FORMAT_CNT",
   "function": {
 "expression": "COUNT_DISTINCT",
@@ -263,8 +250,7 @@
   "gmv_min",
   "gmv_max",
   "trans_cnt",
-  "item_count_sum",
-  "seller_cnt_long"
+  "item_count_sum"
 ]
   }
 ]

http://git-wip-us.apache.org/repos/asf/kylin/blob/623e70f4/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
--
diff --git 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 7e6934b..2f314f0 100644
--- 
a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ 
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -151,19 +151,6 @@
 },
 {
   "id": 7,
-  "name": "SELLER_CNT_LONG",
-  "function": {
-"expression": "COUNT_DISTINCT",
-"parameter": {
-  "type": "column",
-  "value": "SELLER_ID"
-},
-"returntype": "bigint"
-  },
-  "dependent_measure_ref": "SELLER_CNT_HLL"
-},
-{
-  "id": 8,
   "name": "SELLER_FORMAT_CNT",
   "function": {
 "expression": "COUNT_DISTINCT",
@@ -263,8 +250,7 @@
   "gmv_min",
   "gmv_max",
   "trans_cnt",
-  "item_count_sum",
-  "seller_cnt_long"
+  "item_count_sum"
 ]
   }
 ]

[41/50] [abbrv] kylin git commit: KYLIN-976 Support Custom Aggregation Types

2015-12-22 Thread liyang

KYLIN-976 Support Custom Aggregation Types


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c721d679
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c721d679
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c721d679

Branch: refs/heads/1.x-HBase1.1.3
Commit: c721d679727b0345a7fcdd8335a088e7c336aacc
Parents: cdea731
Author: honma 
Authored: Fri Dec 11 16:08:59 2015 +0800
Committer: honma 
Committed: Tue Dec 15 16:50:58 2015 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  40 +-
 .../common/hll/HyperLogLogPlusCounter.java  |  87 +++-
 .../org/apache/kylin/common/util/BytesUtil.java |  16 +
 .../apache/kylin/common/util/Dictionary.java| 199 +
 .../kylin/cube/CubeCapabilityChecker.java   | 147 ---
 .../org/apache/kylin/cube/CubeInstance.java |  20 +-
 .../java/org/apache/kylin/cube/CubeManager.java |   6 +-
 .../java/org/apache/kylin/cube/CubeSegment.java |  14 +-
 .../cube/estimation/CubeSizeEstimationCLI.java  |   4 +-
 .../kylin/cube/kv/AbstractRowKeyEncoder.java|   2 +-
 .../apache/kylin/cube/kv/RowKeyColumnIO.java|   2 +-
 .../apache/kylin/cube/kv/RowKeyColumnOrder.java |   2 +-
 .../apache/kylin/cube/kv/RowValueDecoder.java   |  27 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |  61 ++-
 .../apache/kylin/cube/model/v1/CubeDesc.java|   9 -
 .../model/validation/rule/FunctionRule.java |  33 +-
 .../kylin/cube/DictionaryManagerTest.java   |   2 +-
 .../kylin/cube/kv/RowValueDecoderTest.java  |   6 +-
 .../metadata/measure/MeasureCodecTest.java  |   3 +-
 cube/src/test/resources/data/TEST1_desc.json|   7 +-
 cube/src/test/resources/data/TEST2_desc.json|   7 +-
 cube/src/test/resources/data/TEST3_desc.json| 418 +++
 .../apache/kylin/dict/DateStrDictionary.java|   1 +
 .../java/org/apache/kylin/dict/Dictionary.java  | 199 -
 .../apache/kylin/dict/DictionaryGenerator.java  |  91 ++--
 .../org/apache/kylin/dict/DictionaryInfo.java   |   1 +
 .../kylin/dict/DictionaryInfoSerializer.java|   1 +
 .../apache/kylin/dict/DictionaryManager.java|   1 +
 .../java/org/apache/kylin/dict/ISegment.java|   3 +-
 .../dict/MultipleDictionaryValueEnumerator.java |   1 +
 .../org/apache/kylin/dict/TrieDictionary.java   |   1 +
 .../apache/kylin/dict/lookup/SnapshotTable.java |   2 +-
 .../test_kylin_cube_without_slr_desc.json   |   7 +-
 ...t_kylin_cube_without_slr_left_join_desc.json |   7 +-
 .../invertedindex/IICapabilityChecker.java  | 182 
 .../apache/kylin/invertedindex/IIInstance.java  |  16 +-
 .../apache/kylin/invertedindex/IIManager.java   |   2 +-
 .../apache/kylin/invertedindex/IISegment.java   |   4 +-
 .../invertedindex/index/BitMapContainer.java|   2 +-
 .../index/CompressedValueContainer.java |   2 +-
 .../invertedindex/index/RawTableRecord.java |   2 +-
 .../invertedindex/index/TableRecordInfo.java|   2 +-
 .../index/TableRecordInfoDigest.java|   2 +-
 .../kylin/invertedindex/model/IIDesc.java   |   4 +
 .../invertedindex/IIInstanceTest.java   |   2 +-
 .../invertedindex/InvertedIndexLocalTest.java   |   2 +-
 .../kylin/job/hadoop/cube/BaseCuboidMapper.java |  76 ++--
 .../kylin/job/hadoop/cube/CubeHFileMapper.java  |   4 +-
 .../apache/kylin/job/hadoop/cube/CuboidJob.java |   5 +-
 .../kylin/job/hadoop/cube/CuboidReducer.java|   6 +-
 .../job/hadoop/cube/MergeCuboidMapper.java  |  57 ++-
 .../job/hadoop/cube/NewBaseCuboidMapper.java| 346 ---
 .../kylin/job/dataGen/FactTableGenerator.java   |   2 +-
 .../job/hadoop/cube/BaseCuboidMapperTest.java   |   7 +-
 .../job/hadoop/cube/CubeHFileMapper2Test.java   |   2 +-
 .../kylin/job/hadoop/cube/CubeReducerTest.java  |   2 +-
 .../job/hadoop/cube/MergeCuboidMapperTest.java  |   2 +-
 .../job/hadoop/cube/NDCuboidMapperTest.java |  17 +-
 .../apache/kylin/measure/MeasureAggregator.java |  78 
 .../kylin/measure/MeasureAggregators.java   |  81 
 .../org/apache/kylin/measure/MeasureCodec.java  |  79 
 .../apache/kylin/measure/MeasureIngester.java   |  48 +++
 .../org/apache/kylin/measure/MeasureType.java   | 146 +++
 .../kylin/measure/MeasureTypeFactory.java   |  93 +
 .../kylin/measure/basic/BasicMeasureType.java   | 152 +++
 .../kylin/measure/basic/BigDecimalIngester.java |  41 ++
 .../measure/basic/BigDecimalMaxAggregator.java  |  56 +++
 .../measure/basic/BigDecimalMinAggregator.java  |  57 +++
 .../measure/basic/BigDecimalSumAggregator.java  |  53 +++
 .../kylin/measure/basic/DoubleIngester.java |  46 ++
 .../measure/basic/DoubleMaxAggregator.java  |  55 +++
 .../measure/basic/DoubleMinAggregator.java  |  55 +++
 .../measure/basic/DoubleSumAggregator.java  |  52 +++
 .../kylin/measure/basic/LongIngester.java   |  46 ++

[45/50] [abbrv] kylin git commit: Merge branch 'v1.2-release-fourth-try' into 1.x-staging

2015-12-22 Thread liyang

Merge branch 'v1.2-release-fourth-try' into 1.x-staging


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11946ae0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11946ae0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11946ae0

Branch: refs/heads/1.x-HBase1.1.3
Commit: 11946ae0db1add87f52c8e1ecad9a2ff5bf5d29e
Parents: 7f99c3b 390c4e7
Author: shaofengshi 
Authored: Tue Dec 22 16:28:47 2015 +0800
Committer: shaofengshi 
Committed: Tue Dec 22 16:28:47 2015 +0800

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 2 +-
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)
--

[03/50] [abbrv] kylin git commit: KYLIN-980 spill to disk when sys available memory is low

2015-12-22 Thread liyang

KYLIN-980 spill to disk when sys available memory is low


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f8fc239
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f8fc239
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f8fc239

Branch: refs/heads/1.x-HBase1.1.3
Commit: 0f8fc239162bbca913b1eceb380d89f674928400
Parents: 4c44080
Author: shaofengshi 
Authored: Wed Nov 25 14:38:14 2015 +0800
Committer: shaofengshi 
Committed: Wed Nov 25 14:40:07 2015 +0800

--
 .../common/util/MemoryBudgetController.java | 249 +++
 .../hadoop/cube/FactDistinctColumnsReducer.java |  33 ++-
 2 files changed, 274 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0f8fc239/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java 
b/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
new file mode 100644
index 000..4715ef6
--- /dev/null
+++ 
b/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -0,0 +1,249 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements. See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class MemoryBudgetController {
+
+private static final boolean debug = true;
+
+public interface MemoryConsumer {
+// return number MB released
+int freeUp(int mb);
+}
+
+@SuppressWarnings("serial")
+public static class NotEnoughBudgetException extends IllegalStateException 
{
+
+public NotEnoughBudgetException() {
+super();
+}
+
+public NotEnoughBudgetException(Throwable cause) {
+super(cause);
+}
+}
+
+private static class ConsumerEntry {
+final MemoryConsumer consumer;
+int reservedMB;
+
+ConsumerEntry(MemoryConsumer consumer) {
+this.consumer = consumer;
+}
+}
+
+public static final MemoryBudgetController ZERO_BUDGET = new 
MemoryBudgetController(0);
+public static final int ONE_MB = 1024 * 1024;
+public static final long ONE_GB = 1024 * 1024 * 1024;
+
+private static final Logger logger = 
LoggerFactory.getLogger(MemoryBudgetController.class);
+
+// all budget numbers are in MB
+private final int totalBudgetMB;
+private final ConcurrentHashMap booking = 
new ConcurrentHashMap();
+private int totalReservedMB;
+private final ReentrantLock lock = new ReentrantLock();
+
+public MemoryBudgetController(int totalBudgetMB) {
+Preconditions.checkArgument(totalBudgetMB >= 0);
+Preconditions.checkState(totalBudgetMB <= getSystemAvailMB());
+this.totalBudgetMB = totalBudgetMB;
+this.totalReservedMB = 0;
+}
+
+public int getTotalBudgetMB() {
+return totalBudgetMB;
+}
+
+public int getTotalReservedMB() {
+lock.lock();
+try {
+return totalReservedMB;
+} finally {
+lock.unlock();
+}
+}
+
+public int getRemainingBudgetMB() {
+lock.lock();
+try {
+return totalBudgetMB - totalReservedMB;
+} finally {
+lock.unlock();
+}
+}
+
+public void reserveInsist(MemoryConsumer consumer, int requestMB) {
+long waitStart = 0;
+while (true) {
+try {
+reserve(consumer, requestMB);
+if (debug && waitStart > 0)
+logger.debug(consumer + " waited " + 
(System.currentTimeMillis() - waitStart) + " ms on the " + requestMB + " MB

[18/50] [abbrv] kylin git commit: KYLIN-1168 bug fix

2015-12-22 Thread liyang

KYLIN-1168 bug fix

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95bc7dfa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95bc7dfa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95bc7dfa

Branch: refs/heads/1.x-HBase1.1.3
Commit: 95bc7dfa6d892ac01000f45152f45979b5150c7f
Parents: 62012ad
Author: shaofengshi 
Authored: Fri Dec 4 17:07:16 2015 +0800
Committer: shaofengshi 
Committed: Fri Dec 4 17:07:16 2015 +0800

--
 .../src/main/java/org/apache/kylin/rest/service/CacheService.java  | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/95bc7dfa/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 398577d..38d7400 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -90,8 +90,6 @@ public class CacheService extends BasicService {
 // new data model
 getMetadataManager().reloadDataModelDesc(cacheKey);
 }
-IIDescManager.clearCache();
-CubeDescManager.clearCache();
 break;
 case ALL:
 MetadataManager.clearCache();

[12/50] [abbrv] kylin git commit: upgrade website content from incubator-kylin to kylin

2015-12-22 Thread liyang

upgrade website content from incubator-kylin to kylin


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/316bc048
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/316bc048
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/316bc048

Branch: refs/heads/1.x-HBase1.1.3
Commit: 316bc048376618577885857496a89e2c57aba7fd
Parents: 7888c87
Author: honma 
Authored: Wed Dec 2 16:03:24 2015 +0800
Committer: honma 
Committed: Wed Dec 2 16:03:24 2015 +0800

--
 README.md   | 10 ++--
 docs/README.md  |  2 +-
 website/README.md   |  6 +--
 website/_config.yml |  4 +-
 website/_config_prod.yml|  2 +-
 website/_dev/dev_env.md |  2 +-
 website/_dev/howto_docs.md  |  6 +--
 website/_dev/howto_package.md   |  2 +-
 website/_dev/howto_release.md   | 48 ++--
 website/_dev/index.md   |  6 +--
 website/_docs/index.md  |  2 +-
 website/_docs/install/index.cn.md   |  2 +-
 website/_docs/install/index.md  |  2 +-
 website/_docs/release_notes.md  | 16 +++
 website/_includes/footer.html   | 42 -
 website/_includes/head.cn.html  |  6 +--
 website/_includes/head.html |  6 +--
 website/_includes/header.cn.html|  2 +-
 website/_includes/header.html   |  2 +-
 .../2015-06-10-release-v0.7.1-incubating.md |  4 +-
 website/_posts/blog/2015-08-15-fast-cubing.md   |  4 +-
 .../blog/2015-09-06-release-v1.0-incubating.md  |  4 +-
 .../blog/2015-09-09-fast-cubing-on-spark.md |  2 +-
 .../2015-10-25-release-v1.1-incubating.cn.md|  4 +-
 .../blog/2015-10-25-release-v1.1-incubating.md  |  4 +-
 website/community/index.md  | 10 ++--
 website/community/poweredby.md  |  2 +-
 website/download/index.cn.md| 23 +-
 website/download/index.md   | 22 -
 29 files changed, 121 insertions(+), 126 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/316bc048/README.md
--
diff --git a/README.md b/README.md
index 73313f5..176d89c 100644
--- a/README.md
+++ b/README.md
@@ -5,20 +5,20 @@ Apache Kylin
 
 Apache Kylin is an open source Distributed Analytics Engine, contributed by 
eBay Inc., provides SQL interface and multi-dimensional analysis (OLAP) on 
Hadoop supporting extremely large datasets.
 
-For more details, see the website 
[http://kylin.incubator.apache.org](http://kylin.incubator.apache.org).
+For more details, see the website 
[http://kylin.apache.org](http://kylin.apache.org).
 
 Documentation
 =
-Please refer to 
[http://kylin.incubator.apache.org/docs/](http://kylin.incubator.apache.org/docs/).
+Please refer to [http://kylin.apache.org/docs/](http://kylin.apache.org/docs/).
 
 Get Help
 
-The fastest way to get response from our developers is to send email to our 
mail list ,   
-and remember to subscribe our mail list via 

+The fastest way to get response from our developers is to send email to our 
mail list ,   
+and remember to subscribe our mail list via 
 
 Licnese
 
-Please refer to 
[LICENSE](https://github.com/apache/incubator-kylin/blob/master/LICENSE) file.
+Please refer to [LICENSE](https://github.com/apache/kylin/blob/master/LICENSE) 
file.
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/316bc048/docs/README.md
--
diff --git a/docs/README.md b/docs/README.md
index 166f9fd..22b84f9 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,6 +1,6 @@
 # Apache Kylin Documentation
 
 __All documentations have been moved to Kylin website at:__  
-[http://kylin.incubator.apache.org/docs](http://kylin.incubator.apache.org/docs)
+[http://kylin.apache.org/docs](http://kylin.apache.org/docs)
 
 Please visit there for furthe reference.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/316bc048/website/README.md
--
diff --git a/website/README.md b/website/README.md
index 873ea58..4f0f8a2 100644
--- a/website/README.md
+++ b/website/README.md
@@ -19,7 +19,7 @@ limitations under the License.
 
 # Apache Kylin website  
 This directory contains the source code for

[07/50] [abbrv] kylin git commit: KYLIN-1168 fix UT

2015-12-22 Thread liyang

KYLIN-1168 fix UT

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f5b1561
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f5b1561
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f5b1561

Branch: refs/heads/1.x-HBase1.1.3
Commit: 0f5b1561712b290a00c100e00faf97f430742698
Parents: 04c6607
Author: shaofengshi 
Authored: Thu Nov 26 11:27:56 2015 +0800
Committer: shaofengshi 
Committed: Thu Nov 26 11:27:56 2015 +0800

--
 .../org/apache/kylin/rest/controller/CubeController.java |  1 +
 .../java/org/apache/kylin/rest/service/CacheService.java |  7 +--
 .../org/apache/kylin/rest/service/CacheServiceTest.java  | 11 ---
 3 files changed, 10 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5b1561/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8565d5e..f27f312 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -414,6 +414,7 @@ public class CubeController extends BasicController {
 if (updateModelSuccess == true && updateCubeSuccess == false ) {
 // recover data model
 try {
+oldModelDesc.setLastModified(modelDesc.getLastModified());
 metadataManager.updateDataModelDesc(oldModelDesc);
 } catch (IOException e) {
 logger.error("Failed to recover data model desc:" + 
e.getLocalizedMessage(), e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5b1561/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index c953b7e..7a3238c 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -59,8 +59,10 @@ public class CacheService extends BasicService {
 cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
 break;
 case CUBE_DESC:
-String modelName = 
getCubeDescManager().getCubeDesc(cacheKey).getModelName();
-getMetadataManager().reloadDataModelDesc(modelName);
+if (getCubeDescManager().getCubeDesc(cacheKey) != null) {
+String modelName = 
getCubeDescManager().getCubeDesc(cacheKey).getModelName();
+getMetadataManager().reloadDataModelDesc(modelName);
+}
 getCubeDescManager().reloadCubeDesc(cacheKey);
 IIDescManager.clearCache();
 CubeDescManager.clearCache();
@@ -143,4 +145,5 @@ public class CacheService extends BasicService {
 throw new RuntimeException("error " + log, e);
 }
 }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5b1561/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java 
b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 0436dc9..44755d4 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -47,17 +47,13 @@ import 
org.apache.kylin.rest.broadcaster.BroadcasterReceiveServlet;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
 import org.slf4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
 
 /**
- * Created by qianzhou on 1/16/15.
  */
-
+@Ignore ("Not working for some time")
 public class CacheServiceTest extends LocalFileMetadataTestCase {
 
 private static Server server;
@@ -281,6 +277,7 @@ public class CacheServiceTest extends 
LocalFileMetadataTestCase {
 }
 
 @Test
+//@Ignore ("disable this as only data model change will not trigger a 
broad cast event, see KYLIN-1168")
 public void testMetaCRUD()

[28/50] [abbrv] kylin git commit: KYLIN-1215 minor, update website member's info on community page

2015-12-22 Thread liyang

KYLIN-1215 minor, update website member's info on community page


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8d7d43d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8d7d43d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8d7d43d

Branch: refs/heads/1.x-HBase1.1.3
Commit: b8d7d43d6e4969a0b856f9eaf33c7c499c9747f8
Parents: d5f3d6c
Author: Xiaoyu Wang 
Authored: Wed Dec 9 13:07:10 2015 +0800
Committer: Xiaoyu Wang 
Committed: Wed Dec 9 13:07:10 2015 +0800

--
 website/_data/contributors.yml | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d7d43d/website/_data/contributors.yml
--
diff --git a/website/_data/contributors.yml b/website/_data/contributors.yml
index 41145f3..d32b93f 100644
--- a/website/_data/contributors.yml
+++ b/website/_data/contributors.yml
@@ -91,28 +91,28 @@
   org: eBay
   role: PMC
 - name: Dayue Gao (é«å¤§æ)
-  apacheId: 
+  apacheId: gaodayue
   githubId: gaodayue
   org: Meituan
   role: PMC
 - name: Hua Huang (é»æ¡¦)
-  apacheId: 
+  apacheId: hhuang
   githubId: superhua
   org: MiningLAMP
-  role: PMC 
+  role: PMC
 - name: Dong Li (ææ )
-  apacheId: lidong 
+  apacheId: lidong
   githubId: lidongsjtu
   org: eBay
-  role: committer   
+  role: committer
 - name: Xiaoyu Wang (çæé¨)
   apacheId: wangxiaoyu
-  githubId: 
+  githubId: xiaowangyu
   org: JD.com
-  role: committer 
+  role: committer
 - name: Yerui Sun (åå¶é)
   apacheId: sunyerui
-  githubId: 
+  githubId: sunyerui
   org: Meituan
   role: PMC

[21/50] [abbrv] kylin git commit: KYLIN-1205 Make HBase scan parameters configurable and allow reset of HConnection

2015-12-22 Thread liyang

KYLIN-1205 Make HBase scan parameters configurable and allow reset of 
HConnection


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0c5fd3b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0c5fd3b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0c5fd3b

Branch: refs/heads/1.x-HBase1.1.3
Commit: b0c5fd3bc7cce69dd5f0413d172fd7c7a79d9f5a
Parents: e85b243
Author: Li, Yang 
Authored: Tue Dec 8 10:43:03 2015 +0800
Committer: Li, Yang 
Committed: Tue Dec 8 10:43:03 2015 +0800

--
 .../org/apache/kylin/common/KylinConfig.java |  8 
 .../common/persistence/HBaseConnection.java  |  4 
 .../apache/kylin/rest/service/CacheService.java  |  2 ++
 .../storage/hbase/CubeSegmentTupleIterator.java  | 19 +++
 4 files changed, 29 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 9645b81..7816487 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -499,7 +499,15 @@ public class KylinConfig {
 public int getHBaseKeyValueSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", 
"10485760"));
 }
+
+public int getHBaseScanCacheRows() {
+return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
+}
 
+public int getHBaseScanMaxResultSize() {
+return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
+}
+
 public String getHbaseDefaultCompressionCodec() {
 return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC, "");
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
index dcefc24..9c86376 100644
--- 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
+++ 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
@@ -58,6 +58,10 @@ public class HBaseConnection {
 }
 });
 }
+
+public static void clearCache() {
+ConnPool.clear();
+}
 
 public static HConnection get(String url) {
 // find configuration

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 38d7400..2328592 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -21,6 +21,7 @@ package org.apache.kylin.rest.service;
 import java.io.IOException;
 import java.util.List;
 
+import org.apache.kylin.common.persistence.HBaseConnection;
 import org.apache.kylin.common.restclient.Broadcaster;
 import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.CubeManager;
@@ -101,6 +102,7 @@ public class CacheService extends BasicService {
 RealizationRegistry.clearCache();
 ProjectManager.clearCache();
 BasicService.resetOLAPDataSources();
+HBaseConnection.clearCache();
 break;
 default:
 throw new RuntimeException("invalid cacheType:" + cacheType);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
--
diff --git 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
index 9efbb79..fa4ccd7 100644
--- 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
+++ 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.filter.Filter;
 import

[11/50] [abbrv] kylin git commit: KYLIN-1190 Make memory budget per query configurable

2015-12-22 Thread liyang

KYLIN-1190 Make memory budget per query configurable

Signed-off-by: Li, Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7888c87f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7888c87f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7888c87f

Branch: refs/heads/1.x-HBase1.1.3
Commit: 7888c87fa190e950227eee80d59f8a108b93a18f
Parents: e280002
Author: lidongsjtu 
Authored: Tue Dec 1 16:32:17 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 2 11:11:59 2015 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfig.java|  4 
 .../org/apache/kylin/storage/hbase/CubeStorageEngine.java | 10 +++---
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7888c87f/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index a8f7c5d..9645b81 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -492,6 +492,10 @@ public class KylinConfig {
 return 
Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true"));
 }
 
+public long getQueryMemBudget() {
+return Long.parseLong(this.getOptional("kylin.query.mem.budget", 
String.valueOf(3L * 1024 * 1024 * 1024)));
+}
+
 public int getHBaseKeyValueSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", 
"10485760"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7888c87f/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
--
diff --git 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index ed12781..bbf088e 100644
--- 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -76,7 +76,6 @@ public class CubeStorageEngine implements IStorageEngine {
 private static final Logger logger = 
LoggerFactory.getLogger(CubeStorageEngine.class);
 
 private static final int MERGE_KEYRANGE_THRESHOLD = 100;
-private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; 
// 3G
 
 private final CubeInstance cubeInstance;
 private final CubeDesc cubeDesc;
@@ -643,8 +642,13 @@ public class CubeStorageEngine implements IStorageEngine {
 }
 }
 
-long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
-context.setThreshold((int) rowEst);
+long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / 
rowSizeEst;
+if (rowEst > 0) {
+logger.info("Memory budget is set to: " + rowEst);
+context.setThreshold((int) rowEst);
+} else {
+logger.info("Memory budget is not set.");
+}
 }
 
 private void setLimit(TupleFilter filter, StorageContext context) {

[39/50] [abbrv] kylin git commit: KYLIN-976 Support Custom Aggregation Types

2015-12-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/c721d679/metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
--
diff --git a/metadata/src/main/java/org/apache/kylin/measure/MeasureType.java 
b/metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
new file mode 100644
index 000..66ca209
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
+
+/**
+ * MeasureType captures how a kind of aggregation is defined, how it is 
calculated 
+ * during cube build, and how it is involved in query and storage scan.
+ *
+ * @param  the Java type of aggregation data object, e.g. 
HyperLogLogPlusCounter
+ */
+abstract public class MeasureType {
+
+/* 

+ * Define
+ * 
 */
+
+/** Validates a user defined FunctionDesc has expected parameter etc. 
Throw IllegalArgumentException if anything wrong. */
+public void validate(FunctionDesc functionDesc) throws 
IllegalArgumentException {
+return;
+}
+
+/** Although most aggregated object takes only 8 bytes like long or 
double, 
+ * some advanced aggregation like HyperLogLog or TopN can consume more 
than 10 KB for 
+ * each object, which requires special care on memory allocation. */
+public boolean isMemoryHungry() {
+return false;
+}
+
+/* 

+ * Build
+ * 
 */
+
+/** Return a MeasureIngester which knows how to init aggregation object 
from raw records. */
+abstract public MeasureIngester newIngester();
+
+/** Return a MeasureAggregator which does aggregation. */
+abstract public MeasureAggregator newAggregator();
+
+/** Some special measures need dictionary to encode column values for 
optimal storage. TopN is an example. */
+public List getColumnsNeedDictionary(FunctionDesc functionDesc) 
{
+return Collections.emptyList();
+}
+
+/* 

+ * Cube Selection
+ * 
 */
+
+/**
+ * Some special measures hold columns which are usually treated as 
dimensions (or vice-versa). 
+ * This is where they override to influence cube capability check.
+ *
+ * A SQLDigest contains dimensions and measures extracted from a query. 
After comparing to
+ * cube definition, the matched dimensions and measures are crossed out, 
and what's left is
+ * the unmatchedDimensions and 
unmatchedAggregations.
+ *
+ * Each measure type on the cube is then called on this method to check if 
any of the unmatched
+ * can be fulfilled. If a measure type cannot fulfill any of the 
unmatched, it simply return null.
+ * Or otherwise, unmatchedDimensions and 
unmatchedAggregations must
+ * be modified to drop the satisfied dimension or measure, and a 
CapabilityInfluence object
+ * must be returned to mark the contribution of this measure type.
+ */
+public CapabilityInfluence influenceCapabilityCheck(Collection 
unmatchedDimensions, Collection unmatchedAggregations, SQLDigest 
digest, MeasureDesc measureDesc) {
+return null;
+}
+
+/*

[04/50] [abbrv] kylin git commit: KYLIN-1160 Set default logging configuration in JDBC and Backend code

2015-12-22 Thread liyang

KYLIN-1160 Set default logging configuration in JDBC and Backend code

Signed-off-by: Li, Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b7b6793f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b7b6793f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b7b6793f

Branch: refs/heads/1.x-HBase1.1.3
Commit: b7b6793f814eff6eda6bff6eaef22106e11b789c
Parents: 2b0c3a5
Author: lidongsjtu 
Authored: Tue Nov 24 14:06:46 2015 +0800
Committer: Li, Yang 
Committed: Wed Nov 25 15:15:35 2015 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  4 +++
 .../kylin/common/util/Log4jConfigurer.java  | 37 
 .../main/java/org/apache/kylin/jdbc/Driver.java |  2 ++
 .../java/org/apache/kylin/jdbc/KylinClient.java |  4 +--
 .../org/apache/kylin/jdbc/KylinConnection.java  |  4 +--
 .../kylin/jdbc/util/DefaultLoggerFactory.java   | 21 ---
 .../util/DefaultSslProtocolSocketFactory.java   |  5 +--
 .../jdbc/util/DefaultX509TrustManager.java  | 13 +++
 .../apache/kylin/jdbc/util/Log4jConfigurer.java | 37 
 9 files changed, 94 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b7b6793f/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index edb2586..a8f7c5d 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -35,6 +35,7 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.restclient.RestClient;
 import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.Log4jConfigurer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -243,12 +244,15 @@ public class KylinConfig {
  * @return
  */
 private static KylinConfig loadKylinConfig() {
+Log4jConfigurer.initLogger();
+
 InputStream is = getKylinPropertiesAsInputSteam();
 if (is == null) {
 throw new IllegalArgumentException("Failed to load kylin config");
 }
 KylinConfig config = new KylinConfig();
 config.reloadKylinConfig(is);
+
 return config;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7b6793f/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java 
b/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
new file mode 100644
index 000..c329ced
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/Log4jConfigurer.java
@@ -0,0 +1,37 @@
+package org.apache.kylin.common.util;
+
+import java.util.Enumeration;
+
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+
+/**
+ * Created by dongli on 11/24/15.
+ */
+public class Log4jConfigurer {
+private final static String DEFAULT_PATTERN_LAYOUT = "L4J [%d{-MM-dd 
HH:mm:ss,SSS}][%p][%c] - %m%n";
+private static boolean INITIALIZED = false;
+
+public static void initLogger() {
+if (!INITIALIZED && !isConfigured()) {
+org.apache.log4j.BasicConfigurator.configure(new 
ConsoleAppender(new PatternLayout(DEFAULT_PATTERN_LAYOUT)));
+INITIALIZED = true;
+}
+}
+
+private static boolean isConfigured() {
+if (LogManager.getRootLogger().getAllAppenders().hasMoreElements()) {
+return true;
+} else {
+Enumeration loggers = LogManager.getCurrentLoggers();
+while (loggers.hasMoreElements()) {
+Logger logger = (Logger) loggers.nextElement();
+if (logger.getAllAppenders().hasMoreElements())
+return true;
+}
+}
+return false;
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7b6793f/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
index b58e8ff..a0074b1 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/Driver.java
@@ -25,6 +25,7 @@ import org.apache.calcite.avatica.AvaticaConnection;
 import

[33/50] [abbrv] kylin git commit: [maven-release-plugin] prepare for next development iteration

2015-12-22 Thread liyang

[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/390c4e7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/390c4e7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/390c4e7a

Branch: refs/heads/1.x-HBase1.1.3
Commit: 390c4e7ad2278563a1a224d7b4090093cd9b55c7
Parents: c2589ae
Author: shaofengshi 
Authored: Tue Dec 15 13:50:39 2015 +0800
Committer: shaofengshi 
Committed: Tue Dec 15 13:50:39 2015 +0800

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 2 +-
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 2833b71..03c926c 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 1e7d5b2..2f01bde 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/cube/pom.xml
--
diff --git a/cube/pom.xml b/cube/pom.xml
index 7c4038f..b4fa9da 100644
--- a/cube/pom.xml
+++ b/cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/dictionary/pom.xml
--
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
index f0b2d13..ce86c66 100644
--- a/dictionary/pom.xml
+++ b/dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/invertedindex/pom.xml
--
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index c8d72b8..7fa3e52 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.2
+1.3-SNAPSHOT
 
 4.0.0
 kylin-invertedindex

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index fc76490..6a1f836 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@

org.apache.kylin
kylin
-   1.2
+   1.3-SNAPSHOT

 


http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/job/pom.xml
--
diff --git a/job/pom.xml b/job/pom.xml
index 23085d0..c6b8b01 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 kylin-job

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/metadata/pom.xml
--
diff --git a/metadata/pom.xml b/metadata/pom.xml
index 5ea8b13..c7c849f 100644
--- a/metadata/pom.xml
+++ b/metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/monitor/pom.xml
--
diff --git a/monitor/pom.xml b/monitor/pom.xml
index b67969d..820934f 100644
--- a/monitor/pom.xml
+++ b/monitor/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index b1c81e2..06a9043 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
 org.apache.kylin
 kylin

[06/50] [abbrv] kylin git commit: KYLIN-1168 fix "Update data model is not allowed! Please create a new cube if needed" error

2015-12-22 Thread liyang

KYLIN-1168 fix "Update data model is not allowed! Please create a new cube if 
needed" error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/04c6607f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/04c6607f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/04c6607f

Branch: refs/heads/1.x-HBase1.1.3
Commit: 04c6607fee3775c7225d19ff55503e38238cecdb
Parents: 55c448b
Author: shaofengshi 
Authored: Wed Nov 25 17:25:14 2015 +0800
Committer: shaofengshi 
Committed: Wed Nov 25 17:25:44 2015 +0800

--
 .../kylin/metadata/model/DataModelDesc.java | 38 ---
 .../kylin/rest/controller/CubeController.java   | 39 +---
 .../apache/kylin/rest/service/CacheService.java |  8 ++--
 .../apache/kylin/rest/service/CubeService.java  |  2 +-
 4 files changed, 31 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/04c6607f/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java 
b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 88fafdf..f8dc54d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -220,44 +220,6 @@ public class DataModelDesc extends RootPersistentEntity {
 }
 }
 
-/**
- * Check whether two data model are compatible or not. Compatible means
- * having the same structure. Tow models could be compatible even they
- * have different UUID or last modified time.
- * @param that model to compare with
- * @return true if compatible, false otherwise.
- */
-public boolean compatibleWith(DataModelDesc that) {
-if (this == that)
-return true;
-
-if (that == null)
-return false;
-
-try {
-String thisRepr = excludeHeaderInfo(this);
-String thatRepr = excludeHeaderInfo(that);
-return StringUtils.equals(thisRepr, thatRepr);
-
-} catch (IOException e) {
-logger.error("Failed to serialize DataModelDesc to string", e);
-return false;
-}
-}
-
-private String excludeHeaderInfo(DataModelDesc modelDesc) throws 
IOException {
-// make a copy
-String repr = JsonUtil.writeValueAsString(modelDesc);
-DataModelDesc copy = JsonUtil.readValue(repr, DataModelDesc.class);
-
-copy.setUuid(null);
-copy.setLastModified(0);
-copy.setCapacity(RealizationCapacity.MEDIUM);
-copy.setFilterCondition(null);
-copy.setPartitionDesc(null);
-return JsonUtil.writeValueAsString(copy);
-}
-
 @Override
 public String toString() {
 return "DataModelDesc [name=" + name + "]";

http://git-wip-us.apache.org/repos/asf/kylin/blob/04c6607f/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 9d1dfad..8565d5e 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -359,8 +359,8 @@ public class CubeController extends BasicController {
 }
 
 MetadataManager metadataManager = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
-// KYLIN-958: disallow data model structure change
 DataModelDesc modelDesc = null;
+DataModelDesc oldModelDesc = null;
 if (StringUtils.isNotEmpty(cubeRequest.getModelDescData())) {
 modelDesc = deserializeDataModelDesc(cubeRequest);
 if (modelDesc == null) {
@@ -373,15 +373,11 @@ public class CubeController extends BasicController {
 return errorRequest(cubeRequest, "CubeDesc.model_name " + 
desc.getModelName() + " not consistent with model " + modeName);
 }
 
-DataModelDesc oldModelDesc = 
metadataManager.getDataModelDesc(modeName);
+oldModelDesc = metadataManager.getDataModelDesc(modeName);
 if (oldModelDesc == null) {
 return errorRequest(cubeRequest, "Data model " + modeName + " 
not found");
 }
 
-if (!modelDesc.compatibleWith(oldModelDesc)) {
-return errorRequest(cubeRequest, "Update data model is not 
allowed! Please create a new cube if needed");
-

[40/50] [abbrv] kylin git commit: KYLIN-976 Support Custom Aggregation Types

2015-12-22 Thread liyang

http://git-wip-us.apache.org/repos/asf/kylin/blob/c721d679/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
--
diff --git 
a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java 
b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index e399a70..0ba6566 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -21,15 +21,14 @@ package org.apache.kylin.dict;
 import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.*;
 
-import com.google.common.collect.Lists;
+import com.google.common.collect.Lists;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.*;
 import org.apache.kylin.dict.lookup.ReadableTable;
-import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.datatype.DataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,7 +42,7 @@ public class DictionaryGenerator {
 
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryGenerator.class);
 
-private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
+private static final String[] DATE_PATTERNS = new String[] { "-MM-dd", 
"MMdd" };
 
 private static int getDictionaryMaxCardinality() {
 try {
@@ -53,20 +52,20 @@ public class DictionaryGenerator {
 }
 }
 
-public static Dictionary 
buildDictionaryFromValueEnumerator(DictionaryInfo info, 
IDictionaryValueEnumerator valueEnumerator) throws IOException{
-Dictionary dict = null;
+public static org.apache.kylin.common.util.Dictionary 
buildDictionaryFromValueEnumerator(DictionaryInfo info, 
IDictionaryValueEnumerator valueEnumerator) throws IOException{
+org.apache.kylin.common.util.Dictionary dict = null;
 int baseId = 0; // always 0 for now
-final int nSamples = 5;
-ArrayList samples = Lists.newArrayListWithCapacity(nSamples);
+final int nSamples = 5;
+ArrayList samples = Lists.newArrayListWithCapacity(nSamples);
 
 // build dict, case by data type
 DataType dataType = DataType.getInstance(info.getDataType());
 if (dataType.isDateTimeFamily())
-dict = buildDateStrDict(valueEnumerator, baseId, nSamples, 
samples);
+dict = buildDateStrDict(valueEnumerator, baseId, nSamples, 
samples);
 else if (dataType.isNumberFamily())
-dict = buildNumberDict(valueEnumerator, baseId, nSamples, samples);
+dict = buildNumberDict(valueEnumerator, baseId, nSamples, samples);
 else
-dict = buildStringDict(valueEnumerator, baseId, nSamples, samples);
+dict = buildStringDict(valueEnumerator, baseId, nSamples, samples);
 
 // log a few samples
 StringBuilder buf = new StringBuilder();
@@ -76,54 +75,54 @@ public class DictionaryGenerator {
 
buf.append(s.toString()).append("=>").append(dict.getIdFromValue(s));
 }
 logger.info("Dictionary value samples: " + buf.toString());
-logger.info("Dictionary cardinality: " + dict.getSize());
+logger.info("Dictionary cardinality: " + dict.getSize());
 
-if (dict instanceof TrieDictionary && dict.getSize() > 
DICT_MAX_CARDINALITY)
-throw new IllegalArgumentException("Too high cardinality is not 
suitable for dictionary -- " + info.getSourceTable() + "." + 
info.getSourceColumn() + " cardinality: " + dict.getSize());
+if (dict instanceof TrieDictionary && dict.getSize() > 
DICT_MAX_CARDINALITY)
+throw new IllegalArgumentException("Too high cardinality is not 
suitable for dictionary -- " + info.getSourceTable() + "." + 
info.getSourceColumn() + " cardinality: " + dict.getSize());
 
 return dict;
 }
 
-public static Dictionary mergeDictionaries(DictionaryInfo targetInfo, 
List sourceDicts) throws IOException {
-return buildDictionaryFromValueEnumerator(targetInfo, new 
MultipleDictionaryValueEnumerator(sourceDicts));
+public static org.apache.kylin.common.util.Dictionary 
mergeDictionaries(DictionaryInfo targetInfo, List sourceDicts) 
throws IOException {
+return buildDictionaryFromValueEnumerator(targetInfo, new 
MultipleDictionaryValueEnumerator(sourceDicts));
 }
 
-public static Dictionary buildDictionary(DictionaryInfo info, 
ReadableTable inpTable) throws IOException {
+public static org.apache.kylin.common.util.Dictionary 
buildDictionary(DictionaryInfo info, ReadableTable inpTable) throws IOException 
{
 
 // currently all data types are casted to

[22/50] [abbrv] kylin git commit: KYLIN-1128 clone cube metadata

2015-12-22 Thread liyang

KYLIN-1128 clone cube metadata


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ad77cca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ad77cca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ad77cca

Branch: refs/heads/1.x-HBase1.1.3
Commit: 9ad77cca832dfabb562b8e12d5e28e9e505e245d
Parents: d18b73a
Author: jian 
Authored: Fri Dec 4 12:48:38 2015 +0800
Committer: jian 
Committed: Tue Dec 8 11:03:05 2015 +0800

--
 .../apache/kylin/metadata/MetadataManager.java  | 21 
 .../kylin/rest/controller/CubeController.java   | 15 ++
 webapp/app/js/controllers/cubes.js  |  4 +---
 webapp/app/partials/cubes/cube_clone.html   | 13 
 4 files changed, 33 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ad77cca/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index b540588..800ea44 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -40,6 +40,7 @@ import 
org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.ProjectManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -341,6 +342,26 @@ public class MetadataManager {
 return saveDataModelDesc(desc);
 }
 
+// sync on update
+public DataModelDesc dropModel(DataModelDesc desc) throws IOException {
+logger.info("Dropping model '" + desc.getName() + "'");
+ResourceStore store = getStore();
+if (desc != null)
+store.deleteResource(desc.getResourcePath());
+// clean model cache
+this.afterModelDropped(desc);
+return desc;
+}
+
+private void afterModelDropped(DataModelDesc desc) {
+removeModelCache(desc);
+}
+
+private void removeModelCache(DataModelDesc desc) {
+dataModelDescMap.remove(desc.getName());
+}
+
+
 private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) 
throws IOException {
 dataModelDesc.init(this.getAllTablesMap());
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ad77cca/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b35a35c..95169da 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
@@ -260,7 +261,6 @@ public class CubeController extends BasicController {
 @ResponseBody
 public CubeInstance cloneCube(@PathVariable String cubeName,@RequestBody 
CubeRequest cubeRequest) {
 String targetCubeName = cubeRequest.getCubeName();
-String targetModelName = cubeRequest.getModelDescData();
 String project = cubeRequest.getProject();
 
 CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -275,11 +275,13 @@ public class CubeController extends BasicController {
 
 DataModelDesc modelDesc = metaManager.getDataModelDesc(modelName);
 
-modelDesc.setName(targetModelName);
+//model name same as cube
+modelDesc.setName(targetCubeName);
 modelDesc.setLastModified(0);
 modelDesc.setUuid(UUID.randomUUID().toString());
+DataModelDesc newModel = null;
 try {
-metaManager.createDataModelDesc(modelDesc);
+newModel = metaManager.createDataModelDesc(modelDesc);
 } catch (IOException e) {
 throw new InternalErrorException("failed to clone 
DataModelDesc",e);
 }
@@ -287,11 +289,16 @@ public class CubeController extends BasicController {
 cubeDesc.setName(targetCubeName);
 cubeDesc.setLastModified(0);
 cubeDesc.setUuid(UUID.randomUUID().toString());
-

[25/50] [abbrv] kylin git commit: KYLIN-1203 refresh model timestamp at frontend after backend updated

2015-12-22 Thread liyang

KYLIN-1203 refresh model timestamp at frontend after backend updated


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1c940dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1c940dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1c940dd

Branch: refs/heads/1.x-HBase1.1.3
Commit: a1c940dd5cffd007d62c71e3700ac71df6845973
Parents: 9ad77cc
Author: jian 
Authored: Tue Dec 8 15:12:55 2015 +0800
Committer: jian 
Committed: Tue Dec 8 15:12:55 2015 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 12 
 1 file changed, 12 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a1c940dd/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 2dade99..1178bee 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -289,6 +289,18 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 if ($scope.metaModel.model && 
($scope.metaModel.model.partition_desc.partition_date_start || 
$scope.metaModel.model.partition_desc.partition_date_start == 0)) {
   $scope.metaModel.model.partition_desc.partition_date_start += new 
Date().getTimezoneOffset() * 6;
 }
+
+//update model last modified
+$scope.statusRefresh();
+  }
+
+  $scope.statusRefresh = function(){
+ModelService.get({model_name: $scope.cubeMetaFrame.model_name}, function 
(model) {
+  if (model) {
+//MetaModel.setMetaModel(model);
+$scope.metaModel.model.last_modified = model.last_modified;
+  }
+});
   }
 
   $scope.updateMandatory = function (rowkey_column) {

[15/50] [abbrv] kylin git commit: KYLIN-1168 Update CubeControllerTest and JobControllerTest

2015-12-22 Thread liyang

KYLIN-1168 Update CubeControllerTest and JobControllerTest

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e568c46d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e568c46d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e568c46d

Branch: refs/heads/1.x-HBase1.1.3
Commit: e568c46d9c390db40b3680dcfd7e68b6eeb0d5c6
Parents: 4c7bb8d
Author: shaofengshi 
Authored: Fri Dec 4 11:10:51 2015 +0800
Committer: shaofengshi 
Committed: Fri Dec 4 11:10:51 2015 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java| 2 --
 .../org/apache/kylin/rest/controller/CubeControllerTest.java | 4 
 .../java/org/apache/kylin/rest/controller/JobControllerTest.java | 4 
 3 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e568c46d/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 42cf206..b94ded3 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,7 +27,6 @@ import java.util.Map;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
@@ -43,7 +42,6 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
 import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.exception.NotFoundException;

http://git-wip-us.apache.org/repos/asf/kylin/blob/e568c46d/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java 
b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index bb8cb65..09ce6c5 100644
--- 
a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ 
b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -28,6 +28,7 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.rest.request.CubeRequest;
+import org.apache.kylin.rest.service.AccessService;
 import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.service.JobService;
 import org.apache.kylin.rest.service.ServiceTestBase;
@@ -52,6 +53,8 @@ public class CubeControllerTest extends ServiceTestBase {
 CubeService cubeService;
 @Autowired
 JobService jobService;
+@Autowired
+private AccessService accessService;
 
 @Before
 public void setUp() throws Exception {
@@ -59,6 +62,7 @@ public class CubeControllerTest extends ServiceTestBase {
 cubeController = new CubeController();
 cubeController.setCubeService(cubeService);
 cubeController.setJobService(jobService);
+cubeController.setAccessService(accessService);
 
 cubeDescController = new CubeDescController();
 cubeDescController.setCubeService(cubeService);

http://git-wip-us.apache.org/repos/asf/kylin/blob/e568c46d/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java 
b/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
index fccb294..24de879 100644
--- 
a/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
+++ 
b/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
@@ -33,6 +33,7 @@ import org.apache.kylin.job.dao.ExecutableDao;
 import org.apache.kylin.job.exception.PersistentException;
 import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.request.JobListRequest;
+import org.apache.kylin.rest.service.AccessService;
 import org.apache.kylin.rest.service.CubeService;
 import

[19/50] [abbrv] kylin git commit: update website, add TM mark, added 2 companies on powered by page.

2015-12-22 Thread liyang

update website, add TM mark, added 2 companies on powered by page.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9457552e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9457552e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9457552e

Branch: refs/heads/1.x-HBase1.1.3
Commit: 9457552e07555a67a5b8c0389981cb5eadd59214
Parents: 95bc7df
Author: lukehan 
Authored: Fri Dec 4 22:50:44 2015 +0800
Committer: lukehan 
Committed: Fri Dec 4 22:50:44 2015 +0800

--
 website/_data/contributors.yml | 30 --
 website/community/poweredby.md |  5 +
 website/download/index.cn.md   |  2 +-
 website/download/index.md  |  2 +-
 website/index.cn.md|  7 +++
 website/index.md   |  7 +++
 6 files changed, 37 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/_data/contributors.yml
--
diff --git a/website/_data/contributors.yml b/website/_data/contributors.yml
index dfc3d9b..17b9e38 100644
--- a/website/_data/contributors.yml
+++ b/website/_data/contributors.yml
@@ -39,7 +39,7 @@
   githubId: julianhyde
   org: Hortonworks
   role: Mentor
-- name: Luke Han (é©å¿) 
+- name: Luke Han (é©å¿) (Chair)
   apacheId: lukehan
   githubId: lukehan
   org: eBay
@@ -74,12 +74,12 @@
   apacheId: xduo
   githubId: xduo
   org: Alipay
-  role: PMC
+  role: Emeritus PMC
 - name: Ankur Bansal
   apacheId: abansal
   githubId: abansal
   org: eBay
-  role: PMC
+  role: Emeritus PMC
 - name: Xu Jiang (èæ)
   apacheId: jiangxu
   githubId: jiangxuchina
@@ -94,12 +94,30 @@
   apacheId: 
   githubId: gaodayue
   org: Meituan
-  role: committer
+  role: PMC
 - name: Hua Huang (é»æ¡¦)
   apacheId: 
   githubId: superhua
   org: MiningLAMP
+  role: PMC 
+- name: Dong Li (ææ )
+  apacheId: 
+  githubId: 
+  org: eBay
+  role: committer   
+- name: Xiaoyu Wang (çæé¨)
+  apacheId: 
+  githubId: 
+  org: JD.com
   role: committer 
-
-
+- name: Dong Li (ææ )
+  apacheId: 
+  githubId: lidongsjtu
+  org: eBay
+  role: committer 
+- name: Yerui Sun (åå¶é)
+  apacheId: sunyerui
+  githubId: 
+  org: Meituan
+  role: PMC
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/community/poweredby.md
--
diff --git a/website/community/poweredby.md b/website/community/poweredby.md
index 10d1b93..27448cc 100644
--- a/website/community/poweredby.md
+++ b/website/community/poweredby.md
@@ -25,4 +25,9 @@ __Companies & Organizations__
 * DreamSoft is using Apache Kylin as Big Data Analytics Engine for their 
customers coming from Retail, Manufacturing, Finance and others.
 * [Exponential](http://www.exponential.com)
* Using Kylin as a component in it's data platform LAMBDA to power 
inventory, campaign, behavior and demand analysis for advertising. It has been 
great at handling the size and speed constraints of old gen systems. With the 
first version already in production, we are looking forward to pushing more 
data into kylin and achieve near real time updates. 
+* [Baidu Map](http://map.baidu.com/)  (_NASDAQ: BIDU_)
+   * Apache Kylin is used for analysis Baidu Map's huge amounts of data 
about users, channels and etc, it performs quite fast with its own 
pre-calculation and index technologies along with the cubes.
+* [NetEase](http://www.163.com/)  (_NASDAQ: NTES_)
+   * Apache has been introduced as an OLAP engine for the data platform at 
NetEase, powering data analysis for various products & business with great 
performance.
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/download/index.cn.md
--
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index 81afb2e..ff4cacb 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -4,7 +4,7 @@ title: ä¸è½½
 ---
 
 __ææ°åå¸(æºä»£ç )__  
-ææ°åå¸çApache Kylinå¯ä»¥ä»ASFç½ç«ä¸è½½ï¼:
+ææ°åå¸çApache Kylinâ¢å¯ä»¥ä»ASFç½ç«ä¸è½½ï¼:
 
 * [Apache Kylin 
v1.1.1-incubating](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.1.1-incubating/)
 * [åå¸æ¥å¿](http://kylin.apache.org/docs/release_notes.html)

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/download/index.md
--
diff --git a/website/download/index.md b/website/download/index.md
index e834ccf..eed1a4b 100644
--- a/website/download/index.md
+++ b/website/download/index.md
@@ -5,7 +5,7 @@ permalink: /download/index.html
 ---
 
 __Latest

[17/50] [abbrv] kylin git commit: KYLIN-930 list cube realization under each project

2015-12-22 Thread liyang

KYLIN-930 list cube realization under each project

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/62012ad5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/62012ad5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/62012ad5

Branch: refs/heads/1.x-HBase1.1.3
Commit: 62012ad564b53c2fa666444977fd21cf1f042f00
Parents: eb692f3
Author: jian 
Authored: Fri Dec 4 14:16:09 2015 +0800
Committer: shaofengshi 
Committed: Fri Dec 4 16:15:42 2015 +0800

--
 webapp/app/partials/projects/project_detail.html | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/62012ad5/webapp/app/partials/projects/project_detail.html
--
diff --git a/webapp/app/partials/projects/project_detail.html 
b/webapp/app/partials/projects/project_detail.html
index 35c8ef0..2644f08 100644
--- a/webapp/app/partials/projects/project_detail.html
+++ b/webapp/app/partials/projects/project_detail.html
@@ -32,7 +32,7 @@
 
 
 
-
+
 
 
 Cube Name
@@ -40,14 +40,14 @@
 
 
 
-
-{{cube}}
-Detail
+
+{{item.realization}}
+Detail
 
 
 
 
-
+

[49/50] [abbrv] kylin git commit: KYLIN-920 & KYLIN-782 Upgrade to HBase 1.1 (with help from murkrishn <murkris...@ebay.com>)

2015-12-22 Thread liyang

KYLIN-920 & KYLIN-782 Upgrade to HBase 1.1 (with help from murkrishn 
)

Signed-off-by: Li, Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5a871c60
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5a871c60
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5a871c60

Branch: refs/heads/1.x-HBase1.1.3
Commit: 5a871c607f86d623ae3dc7f42a0c489c6e5d558a
Parents: 0603a19
Author: Yang Li 
Authored: Sun Aug 16 20:22:13 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 23 10:19:13 2015 +0800

--
 .../common/persistence/HBaseConnection.java | 254 
 .../common/persistence/HBaseResourceStore.java  |  31 +-
 .../common/util/HBaseRegionSizeCalculator.java  |  41 +-
 .../kylin/common/util/BasicHadoopTest.java  |  11 +-
 .../kylin/job/cube/GarbageCollectionStep.java   |  22 +-
 .../kylin/job/hadoop/cube/CubeHFileJob.java |  18 +-
 .../job/hadoop/cube/StorageCleanupJob.java  |  26 +-
 .../kylin/job/hadoop/hbase/CreateHTableJob.java |   8 +-
 .../hadoop/invertedindex/IICreateHFileJob.java  |  22 +-
 .../hadoop/invertedindex/IICreateHTableJob.java |  11 +-
 .../apache/kylin/job/tools/CleanHtableCLI.java  |   8 +-
 .../kylin/job/tools/CubeMigrationCLI.java   |  64 +-
 .../kylin/job/tools/DeployCoprocessorCLI.java   | 625 ++-
 .../job/tools/GridTableHBaseBenchmark.java  |  37 +-
 .../kylin/job/tools/HtableAlterMetadataCLI.java |   8 +-
 .../apache/kylin/job/tools/RowCounterCLI.java   |  11 +-
 .../org/apache/kylin/job/ExportHBaseData.java   |  18 +-
 .../kylin/job/hadoop/hbase/TestHbaseClient.java |  13 +-
 .../kylin/job/tools/HBaseRowDigestTest.java |  11 +-
 monitor/pom.xml |   6 +
 .../kylin/monitor/MonitorMetaManager.java   |  49 +-
 pom.xml |  14 +-
 .../apache/kylin/rest/service/AclService.java   |  38 +-
 .../apache/kylin/rest/service/CubeService.java  |  35 +-
 .../apache/kylin/rest/service/QueryService.java |  21 +-
 .../apache/kylin/rest/service/UserService.java  |  27 +-
 .../storage/filter/BitMapFilterEvaluator.java   |   1 -
 .../storage/hbase/CubeSegmentTupleIterator.java |  19 +-
 .../kylin/storage/hbase/CubeStorageEngine.java  |   4 +-
 .../storage/hbase/HBaseClientKVIterator.java| 187 +++---
 .../hbase/InvertedIndexStorageEngine.java   | 114 ++--
 .../kylin/storage/hbase/PingHBaseCLI.java   | 179 +++---
 .../storage/hbase/RegionScannerAdapter.java |  10 +-
 .../hbase/SerializedHBaseTupleIterator.java |   4 +-
 .../endpoint/EndpointTupleIterator.java |  15 +-
 .../hbase/coprocessor/endpoint/IIEndpoint.java  |   2 +-
 .../observer/AggregateRegionObserver.java   |   2 +-
 .../observer/AggregationScanner.java|  14 +-
 .../observer/ObserverAggregationCache.java  |  10 +-
 .../coprocessor/observer/ObserverEnabler.java   |   4 +-
 .../storage/hbase/InvertedIndexHBaseTest.java   | 227 ---
 .../observer/AggregateRegionObserverTest.java   |  72 +--
 .../minicluster/HiveMiniClusterTest.java|   3 +-
 43 files changed, 1151 insertions(+), 1145 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5a871c60/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
index 9c86376..3c07654 100644
--- 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
+++ 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
@@ -1,122 +1,132 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import

[09/50] [abbrv] kylin git commit: Revert "KYLIN-980 spill to disk when sys available memory is low"

2015-12-22 Thread liyang

Revert "KYLIN-980 spill to disk when sys available memory is low"

This reverts commit 0f8fc239162bbca913b1eceb380d89f674928400.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cfe9f0a8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cfe9f0a8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cfe9f0a8

Branch: refs/heads/1.x-HBase1.1.3
Commit: cfe9f0a8ac960b59281aefa0245e79ed4b534294
Parents: 6f93a4d
Author: shaofengshi 
Authored: Mon Nov 30 14:08:27 2015 +0800
Committer: shaofengshi 
Committed: Mon Nov 30 14:08:27 2015 +0800

--
 .../common/util/MemoryBudgetController.java | 249 ---
 .../hadoop/cube/FactDistinctColumnsReducer.java |  33 +--
 2 files changed, 8 insertions(+), 274 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cfe9f0a8/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java 
b/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
deleted file mode 100644
index 4715ef6..000
--- 
a/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements. See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License. You may obtain a copy of the License at
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.kylin.common.util;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReentrantLock;
-
-public class MemoryBudgetController {
-
-private static final boolean debug = true;
-
-public interface MemoryConsumer {
-// return number MB released
-int freeUp(int mb);
-}
-
-@SuppressWarnings("serial")
-public static class NotEnoughBudgetException extends IllegalStateException 
{
-
-public NotEnoughBudgetException() {
-super();
-}
-
-public NotEnoughBudgetException(Throwable cause) {
-super(cause);
-}
-}
-
-private static class ConsumerEntry {
-final MemoryConsumer consumer;
-int reservedMB;
-
-ConsumerEntry(MemoryConsumer consumer) {
-this.consumer = consumer;
-}
-}
-
-public static final MemoryBudgetController ZERO_BUDGET = new 
MemoryBudgetController(0);
-public static final int ONE_MB = 1024 * 1024;
-public static final long ONE_GB = 1024 * 1024 * 1024;
-
-private static final Logger logger = 
LoggerFactory.getLogger(MemoryBudgetController.class);
-
-// all budget numbers are in MB
-private final int totalBudgetMB;
-private final ConcurrentHashMap booking = 
new ConcurrentHashMap();
-private int totalReservedMB;
-private final ReentrantLock lock = new ReentrantLock();
-
-public MemoryBudgetController(int totalBudgetMB) {
-Preconditions.checkArgument(totalBudgetMB >= 0);
-Preconditions.checkState(totalBudgetMB <= getSystemAvailMB());
-this.totalBudgetMB = totalBudgetMB;
-this.totalReservedMB = 0;
-}
-
-public int getTotalBudgetMB() {
-return totalBudgetMB;
-}
-
-public int getTotalReservedMB() {
-lock.lock();
-try {
-return totalReservedMB;
-} finally {
-lock.unlock();
-}
-}
-
-public int getRemainingBudgetMB() {
-lock.lock();
-try {
-return totalBudgetMB - totalReservedMB;
-} finally {
-lock.unlock();
-}
-}
-
-public void reserveInsist(MemoryConsumer consumer, int requestMB) {
-long waitStart = 0;
-while (true) {
-try {
-reserve(consumer, requestMB);
-if (debug && waitStart > 0)
-logger.debug(consumer + " waited " +

svn commit: r1721480 - in /kylin/site: cn/download/index.html download/index.html feed.xml

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Wed Dec 23 01:32:50 2015
New Revision: 1721480

URL: http://svn.apache.org/viewvc?rev=1721480=rev
Log:
update link for archive site

Modified:
kylin/site/cn/download/index.html
kylin/site/download/index.html
kylin/site/feed.xml

Modified: kylin/site/cn/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/cn/download/index.html?rev=1721480=1721479=1721480=diff
==
--- kylin/site/cn/download/index.html (original)
+++ kylin/site/cn/download/index.html Wed Dec 23 01:32:50 2015
@@ -196,7 +196,7 @@
 å¯ä»¥ä»åä¸ªçæ¬æå½åææ°çå¼ååæ¯ä¸çæäºè¿å¶å
ï¼è¯·åèè¿ç¯https://kylin.apache.org/development/howto_package.html;>å¸®å©ææ¡£
 
 ä»¥åççæ¬
-Apache Kylinçæ§çæ¬å¯ä»¥ä»https://dist.apache.org/repos/dist/release/kylin/;>å½æ¡£ä¸ä¸è½½ã
+Apache Kylinçæ§çæ¬å¯ä»¥ä»https://archive.apache.org/dist/kylin/;>å½æ¡£ä¸ä¸è½½ã
 
 ODBC é©±å¨  
 Kylin ODBC é©±å¨è¦æ±é¦åå®è£Microsoft Visual C++ 2012 
Redistributableã 

Modified: kylin/site/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/download/index.html?rev=1721480=1721479=1721480=diff
==
--- kylin/site/download/index.html (original)
+++ kylin/site/download/index.html Wed Dec 23 01:32:50 2015
@@ -200,7 +200,7 @@ Also, please aware this is not a formal
 To build binary package from any version even latest development branch, 
please refer to this https://kylin.apache.org/development/howto_package.html;>guide
 
 Previous Release  
- Older releases may be found in the https://dist.apache.org/repos/dist/release/kylin/;>archives.
+ Older releases may be found in the https://archive.apache.org/dist/kylin/;>archives.
 
 ODBC Driver  
 Kylin ODBC driver requires http://www.microsoft.com/en-us/download/details.aspx?id=30679;>Microsoft 
Visual C++ 2012 Redistributable installed first. 

Modified: kylin/site/feed.xml
URL: 
http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1721480=1721479=1721480=diff
==
--- kylin/site/feed.xml (original)
+++ kylin/site/feed.xml Wed Dec 23 01:32:50 2015
@@ -19,8 +19,8 @@
 Apache Kylin Home
 http://kylin.apache.org/
 http://kylin.apache.org/feed.xml; rel="self" 
type="application/rss+xml"/>
-Tue, 22 Dec 2015 06:13:27 -0800
-Tue, 22 Dec 2015 06:13:27 -0800
+Tue, 22 Dec 2015 17:30:32 -0800
+Tue, 22 Dec 2015 17:30:32 -0800
 Jekyll v2.5.3

kylin git commit: update v1.2 binary package for hbase1.1

2015-12-22 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 0603a19e3 -> 430319316


update v1.2 binary package for hbase1.1


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/43031931
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/43031931
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/43031931

Branch: refs/heads/1.x-staging
Commit: 430319316f40874982964ed35dc8c4fd438c9000
Parents: 0603a19
Author: shaofengshi 
Authored: Wed Dec 23 10:55:30 2015 +0800
Committer: shaofengshi 
Committed: Wed Dec 23 10:55:30 2015 +0800

--
 website/download/index.cn.md | 4 ++--
 website/download/index.md| 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/43031931/website/download/index.cn.md
--
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index 7da5740..b5710d9 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -22,8 +22,8 @@ __äºè¿å¶å for HBase 1.1.3åæ´é«çæ¬__
 è¯·æ³¨ææ¤å®è£åéè¦HBase 
1.1.3ææ´é«çæ¬ï¼ä¹åçæ¬ä¸æä¸ä¸ªå·²ç¥çå³äºfuzzy 
keyè¿æ»¤å¨çç¼ºé·ï¼ä¼å¯¼è´Kylinæ¥è¯¢ç»æç¼ºå°è®°å½: 
[HBASE-14269](https://issues.apache.org/jira/browse/HBASE-14269)
 
æ¤å¤è¯·æ³¨æï¼è¿ä¸æ¯ä¸ä¸ªæ£å¼çåå¸çï¼æ²¡æç»è¿å®æ´çæµè¯ã
 
-  * 
[apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.2-incubating-snapshot/apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz)
-  * Git commit 
[3623dd6ff93d76141bb6a5fb623a3421ae78ca93](https://github.com/apache/kylin/commit/3623dd6ff93d76141bb6a5fb623a3421ae78ca93)
+  * 
[apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz)
+  * Git commit 
[a3b8eb04955310abec158ea30f61deb0119679d1](https://github.com/apache/kylin/commit/a3b8eb04955310abec158ea30f61deb0119679d1)
 
 
 å¯¹äºHBase 1.0ç¨æ·ï¼å»ºè®®æ¨åçº§å°1.1.3æéçº§å°0.98/0.99.
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/43031931/website/download/index.md
--
diff --git a/website/download/index.md b/website/download/index.md
index 98ae1ae..52d3413 100644
--- a/website/download/index.md
+++ b/website/download/index.md
@@ -24,8 +24,8 @@ Note the requirement of HBase version 1.1.3 (or above). There 
is a known bug in
 missing rows or lesser aggregations in Kylin query result: 
[HBASE-14269](https://issues.apache.org/jira/browse/HBASE-14269)
 Also, please aware this is not a formal release, and it is not fully tested:
 
-* 
[apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.2-incubating-snapshot/apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz)
-* Git commit 
[3623dd6ff93d76141bb6a5fb623a3421ae78ca93](https://github.com/apache/kylin/commit/3623dd6ff93d76141bb6a5fb623a3421ae78ca93)
 
+* 
[apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz)
+* Git commit 
[a3b8eb04955310abec158ea30f61deb0119679d1](https://github.com/apache/kylin/commit/a3b8eb04955310abec158ea30f61deb0119679d1)
 
 
 If you're using HBase 1.0, we suggest you to upgrade to 1.1.3+ or downgrade to 
0.98/0.99.

svn commit: r1721485 - in /kylin/site: cn/download/index.html download/index.html feed.xml

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Wed Dec 23 02:55:11 2015
New Revision: 1721485

URL: http://svn.apache.org/viewvc?rev=1721485=rev
Log:
update v1.2 binary package for hbase1.1

Modified:
kylin/site/cn/download/index.html
kylin/site/download/index.html
kylin/site/feed.xml

Modified: kylin/site/cn/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/cn/download/index.html?rev=1721485=1721484=1721485=diff
==
--- kylin/site/cn/download/index.html (original)
+++ kylin/site/cn/download/index.html Wed Dec 23 02:55:11 2015
@@ -186,8 +186,8 @@
 
æ¤å¤è¯·æ³¨æï¼è¿ä¸æ¯ä¸ä¸ªæ£å¼çåå¸çï¼æ²¡æç»è¿å®æ´çæµè¯ã
 
 
-  https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.2-incubating-snapshot/apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz;>apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz
-  Git commit https://github.com/apache/kylin/commit/3623dd6ff93d76141bb6a5fb623a3421ae78ca93;>3623dd6ff93d76141bb6a5fb623a3421ae78ca93
+  https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz;>apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
+  Git commit https://github.com/apache/kylin/commit/a3b8eb04955310abec158ea30f61deb0119679d1;>a3b8eb04955310abec158ea30f61deb0119679d1
 
 
 å¯¹äºHBase 1.0ç¨æ·ï¼å»ºè®®æ¨åçº§å°1.1.3æéçº§å°0.98/0.99.

Modified: kylin/site/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/download/index.html?rev=1721485=1721484=1721485=diff
==
--- kylin/site/download/index.html (original)
+++ kylin/site/download/index.html Wed Dec 23 02:55:11 2015
@@ -190,8 +190,8 @@ missing rows or lesser aggregations in K
 Also, please aware this is not a formal release, and it is not fully 
tested:
 
 
-  https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.2-incubating-snapshot/apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz;>apache-kylin-1.2-HBase1.1-incubating-SNAPSHOT-bin.tar.gz
-  Git commit https://github.com/apache/kylin/commit/3623dd6ff93d76141bb6a5fb623a3421ae78ca93;>3623dd6ff93d76141bb6a5fb623a3421ae78ca93
+  https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.3-snapshot/apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz;>apache-kylin-1.3-HBase-1.1-SNAPSHOT-bin.tar.gz
+  Git commit https://github.com/apache/kylin/commit/a3b8eb04955310abec158ea30f61deb0119679d1;>a3b8eb04955310abec158ea30f61deb0119679d1
 
 
 If youâre using HBase 1.0, we suggest you to upgrade to 1.1.3+ or 
downgrade to 0.98/0.99.

Modified: kylin/site/feed.xml
URL: 
http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1721485=1721484=1721485=diff
==
--- kylin/site/feed.xml (original)
+++ kylin/site/feed.xml Wed Dec 23 02:55:11 2015
@@ -19,8 +19,8 @@
 Apache Kylin Home
 http://kylin.apache.org/
 http://kylin.apache.org/feed.xml; rel="self" 
type="application/rss+xml"/>
-Tue, 22 Dec 2015 17:30:32 -0800
-Tue, 22 Dec 2015 17:30:32 -0800
+Tue, 22 Dec 2015 18:53:34 -0800
+Tue, 22 Dec 2015 18:53:34 -0800
 Jekyll v2.5.3

kylin git commit: KYLIN-1245 Save 'mapper overlap ratio' in cube stats

2015-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 1410d9d30 -> db95d72ca


KYLIN-1245 Save 'mapper overlap ratio' in cube stats


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db95d72c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db95d72c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db95d72c

Branch: refs/heads/2.x-staging
Commit: db95d72caa801151824177c613acc91c5549f478
Parents: 1410d9d
Author: Li, Yang 
Authored: Wed Dec 23 14:02:17 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 23 14:03:25 2015 +0800

--
 build/conf/kylin_hive_conf.xml  |   2 +-
 .../org/apache/kylin/cube/CubeInstance.java |  10 -
 .../java/org/apache/kylin/cube/CubeManager.java |   6 +-
 .../org/apache/kylin/cube/CubeSegmentsTest.java |   2 +-
 .../kylin/engine/mr/common/CubeStatsReader.java | 227 +++
 .../kylin/engine/mr/common/CuboidStatsUtil.java |  14 +-
 .../mr/steps/FactDistinctColumnsReducer.java|  19 +-
 .../kylin/engine/mr/steps/InMemCuboidJob.java   | 153 +
 .../apache/kylin/engine/spark/SparkCubing.java  |   5 +-
 .../streaming/monitor/StreamingMonitor.java |   2 +-
 .../apache/kylin/rest/service/CubeService.java  |   2 +-
 .../apache/kylin/rest/service/JobService.java   |   2 +-
 .../storage/hbase/steps/CreateHTableJob.java|   3 +-
 13 files changed, 267 insertions(+), 180 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db95d72c/build/conf/kylin_hive_conf.xml
--
diff --git a/build/conf/kylin_hive_conf.xml b/build/conf/kylin_hive_conf.xml
index afa53f7..f91f489 100644
--- a/build/conf/kylin_hive_conf.xml
+++ b/build/conf/kylin_hive_conf.xml
@@ -8,7 +8,7 @@
 
 
 dfs.block.size
-10485760
+3200
 Want more mappers for in-mem cubing, thus smaller the DFS 
block size
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/db95d72c/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index dccc3f1..8363a2b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -299,16 +299,6 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return result;
 }
 
-public List getSegment(SegmentStatusEnum status) {
-List result = Lists.newArrayList();
-for (CubeSegment segment : segments) {
-if (segment.getStatus() == status) {
-result.add(segment);
-}
-}
-return result;
-}
-
 public CubeSegment getSegment(String name, SegmentStatusEnum status) {
 for (CubeSegment segment : segments) {
 if ((null != segment.getName() && segment.getName().equals(name)) 
&& segment.getStatus() == status) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/db95d72c/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 88904bb..8e4906b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -271,7 +271,7 @@ public class CubeManager implements IRealizationProvider {
 }
 
 private boolean validateReadySegments(CubeInstance cube) {
-final List readySegments = 
cube.getSegment(SegmentStatusEnum.READY);
+final List readySegments = 
cube.getSegments(SegmentStatusEnum.READY);
 if (readySegments.size() == 0) {
 return true;
 }
@@ -475,7 +475,7 @@ public class CubeManager implements IRealizationProvider {
 }
 
 private Pair alignMergeRange(CubeInstance cube, long 
startDate, long endDate) {
-List readySegments = 
cube.getSegment(SegmentStatusEnum.READY);
+List readySegments = 
cube.getSegments(SegmentStatusEnum.READY);
 if (readySegments.isEmpty()) {
 throw new IllegalStateException("there are no segments in ready 
state");
 }
@@ -610,7 +610,7 @@ public class CubeManager implements IRealizationProvider {
 return null;
 }
 
-List readySegments = 
Lists.newArrayList(cube.getSegment(SegmentStatusEnum.READY));
+List readySegments = 
Lists.newArrayList(cube.getSegments(SegmentStatusEnum.READY));

kylin git commit: KYLIN-1245 bug fix when reading stats seq file

2015-12-22 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging db95d72ca -> 54071e311


KYLIN-1245 bug fix when reading stats seq file


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/54071e31
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/54071e31
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/54071e31

Branch: refs/heads/2.x-staging
Commit: 54071e311d1bfcbc96390631a2d98d3ce70aaef6
Parents: db95d72
Author: Li, Yang 
Authored: Wed Dec 23 15:13:57 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 23 15:13:57 2015 +0800

--
 .../kylin/engine/mr/common/CubeStatsReader.java | 29 +++-
 1 file changed, 22 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/54071e31/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index fc27a81..bbc724a 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.engine.mr.common;
 
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintWriter;
@@ -29,7 +31,7 @@ import java.util.Map;
 import javax.annotation.Nullable;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.LongWritable;
@@ -76,14 +78,14 @@ public class CubeStatsReader {
 public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) 
throws IOException {
 ResourceStore store = ResourceStore.getStore(kylinConfig);
 String statsKey = cubeSegment.getStatisticsResourcePath();
-InputStream is = store.getResource(statsKey).inputStream;
+File tmpSeqFile = 
writeTmpSeqFile(store.getResource(statsKey).inputStream);
 Reader reader = null;
 
 try {
 Configuration hadoopConf = HadoopUtil.getCurrentConfiguration();
 
-Option streamInput = SequenceFile.Reader.stream(new 
FSDataInputStream(is));
-reader = new SequenceFile.Reader(hadoopConf, streamInput);
+Option seqInput = SequenceFile.Reader.file(new Path("file://" + 
tmpSeqFile.getAbsolutePath()));
+reader = new SequenceFile.Reader(hadoopConf, seqInput);
 
 int percentage = 100;
 double mapperOverlapRatio = 0;
@@ -111,10 +113,23 @@ public class CubeStatsReader {
 
 } finally {
 IOUtils.closeStream(reader);
-IOUtils.closeStream(is);
+tmpSeqFile.delete();
 }
 }
 
+private File writeTmpSeqFile(InputStream inputStream) throws IOException {
+File tempFile = File.createTempFile("kylin_stats_tmp", ".seq");
+FileOutputStream out = null;
+try {
+out = new FileOutputStream(tempFile);
+org.apache.commons.io.IOUtils.copy(inputStream, out);
+} finally {
+IOUtils.closeStream(inputStream);
+IOUtils.closeStream(out);
+}
+return tempFile;
+}
+
 public Map getCuboidRowCountMap() {
 return getCuboidRowCountMapFromSampling(cuboidRowCountMap, 
samplingPercentage);
 }
@@ -211,12 +226,12 @@ public class CubeStatsReader {
 logger.info("Cuboid " + cuboidId + " has " + rowCount + " rows, each 
row size is " + bytesLength + " bytes." + " Total size is " + ret + "M.");
 return ret;
 }
-
+
 public static void main(String[] args) throws IOException {
 KylinConfig config = KylinConfig.getInstanceFromEnv();
 CubeInstance cube = CubeManager.getInstance(config).getCube(args[0]);
 List segments = cube.getSegments(SegmentStatusEnum.READY);
-
+
 PrintWriter out = new PrintWriter(System.out);
 for (CubeSegment seg : segments) {
 new CubeStatsReader(seg, config).print(out);

[2/3] kylin git commit: [maven-release-plugin] prepare for next development iteration

2015-12-22 Thread shaofengshi

[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/390c4e7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/390c4e7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/390c4e7a

Branch: refs/heads/1.x-staging
Commit: 390c4e7ad2278563a1a224d7b4090093cd9b55c7
Parents: c2589ae
Author: shaofengshi 
Authored: Tue Dec 15 13:50:39 2015 +0800
Committer: shaofengshi 
Committed: Tue Dec 15 13:50:39 2015 +0800

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 2 +-
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 2833b71..03c926c 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 1e7d5b2..2f01bde 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/cube/pom.xml
--
diff --git a/cube/pom.xml b/cube/pom.xml
index 7c4038f..b4fa9da 100644
--- a/cube/pom.xml
+++ b/cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/dictionary/pom.xml
--
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
index f0b2d13..ce86c66 100644
--- a/dictionary/pom.xml
+++ b/dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/invertedindex/pom.xml
--
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index c8d72b8..7fa3e52 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.2
+1.3-SNAPSHOT
 
 4.0.0
 kylin-invertedindex

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index fc76490..6a1f836 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@

org.apache.kylin
kylin
-   1.2
+   1.3-SNAPSHOT

 


http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/job/pom.xml
--
diff --git a/job/pom.xml b/job/pom.xml
index 23085d0..c6b8b01 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 kylin-job

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/metadata/pom.xml
--
diff --git a/metadata/pom.xml b/metadata/pom.xml
index 5ea8b13..c7c849f 100644
--- a/metadata/pom.xml
+++ b/metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/monitor/pom.xml
--
diff --git a/monitor/pom.xml b/monitor/pom.xml
index b67969d..820934f 100644
--- a/monitor/pom.xml
+++ b/monitor/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2
+1.3-SNAPSHOT
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/390c4e7a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index b1c81e2..06a9043 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
 org.apache.kylin
 kylin

[3/3] kylin git commit: Merge branch 'v1.2-release-fourth-try' into 1.x-staging

2015-12-22 Thread shaofengshi

Merge branch 'v1.2-release-fourth-try' into 1.x-staging


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11946ae0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11946ae0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11946ae0

Branch: refs/heads/1.x-staging
Commit: 11946ae0db1add87f52c8e1ecad9a2ff5bf5d29e
Parents: 7f99c3b 390c4e7
Author: shaofengshi 
Authored: Tue Dec 22 16:28:47 2015 +0800
Committer: shaofengshi 
Committed: Tue Dec 22 16:28:47 2015 +0800

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 2 +-
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)
--

[19/50] kylin git commit: KYLIN-1119 enhance find-hive-dependency to support one common case

2015-12-22 Thread shaofengshi

KYLIN-1119 enhance find-hive-dependency to support one common case


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1cd18964
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1cd18964
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1cd18964

Branch: refs/heads/master
Commit: 1cd189649ad6d559c0d373b1e2b31c55387aa13e
Parents: 0f8fc23
Author: shaofengshi 
Authored: Wed Nov 25 14:38:55 2015 +0800
Committer: shaofengshi 
Committed: Wed Nov 25 14:40:07 2015 +0800

--
 bin/find-hive-dependency.sh | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1cd18964/bin/find-hive-dependency.sh
--
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
index cb5a43e..5994dda 100755
--- a/bin/find-hive-dependency.sh
+++ b/bin/find-hive-dependency.sh
@@ -44,6 +44,8 @@ then
 hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
 if [ -d "${hadoop_home}/hive-hcatalog" ]; then
   hcatalog_home=${hadoop_home}/hive-hcatalog
+elif [ -d "${hadoop_home}/hive/hcatalog" ]; then
+  hcatalog_home=${hadoop_home}/hive/hcatalog
 else 
   echo "Couldn't locate hcatalog installation, please make sure it is 
installed and set HCAT_HOME to the path."
   exit 1

[06/50] kylin git commit: KYLIN-1115 unit test with minicluster doesn't work

2015-12-22 Thread shaofengshi

KYLIN-1115 unit test with minicluster doesn't work 

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01ea9bc9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01ea9bc9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01ea9bc9

Branch: refs/heads/master
Commit: 01ea9bc9137449fc567d558fff64fb66f8cc284b
Parents: ae6f23c
Author: shaofengshi 
Authored: Thu Nov 19 21:38:38 2015 +0800
Committer: shaofengshi 
Committed: Thu Nov 19 21:38:38 2015 +0800

--
 .../kylin/common/persistence/HBaseResourceStore.java | 11 +++
 .../apache/kylin/common/persistence/ResourceStore.java   |  2 +-
 2 files changed, 4 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/01ea9bc9/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
index 35a62b5..23dd831 100644
--- 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
+++ 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseResourceStore.java
@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -88,17 +89,11 @@ public class HBaseResourceStore extends ResourceStore {
 int cut = metadataUrl.indexOf('@');
 tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : 
metadataUrl.substring(0, cut);
 hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+if (!(StringUtils.isEmpty(hbaseUrl) || "hbase".equals(hbaseUrl)))
+throw new IllegalArgumentException("to use hbase storage, pls set 
'kylin.metadata.url=kylin_metadata@hbase' in kylin.properties");
 
 createHTableIfNeeded(getAllInOneTableName());
 
-//tableNameMap = new LinkedHashMap();
-//for (Entry entry : 
TABLE_SUFFIX_MAP.entrySet()) {
-//String pathPrefix = entry.getKey();
-//String tableName = tableNameBase + entry.getValue();
-//tableNameMap.put(pathPrefix, tableName);
-//createHTableIfNeeded(tableName);
-//}
-
 }
 
 private void createHTableIfNeeded(String tableName) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/01ea9bc9/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java 
b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 5375597..505c72a 100644
--- 
a/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -61,8 +61,8 @@ abstract public class ResourceStore {
 public static final ArrayList knownImpl = 
new ArrayList();
 
 static {
-knownImpl.add(HBaseResourceStore.class);
 knownImpl.add(FileResourceStore.class);
+knownImpl.add(HBaseResourceStore.class);
 }
 
 public static ResourceStore getStore(KylinConfig kylinConfig) {

svn commit: r11696 - in /release/kylin: apache-kylin-0.7.1-incubating/ apache-kylin-0.7.2-incubating/

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Tue Dec 22 08:06:56 2015
New Revision: 11696

Log:
Remove old 0.7.x release

Removed:
release/kylin/apache-kylin-0.7.1-incubating/
release/kylin/apache-kylin-0.7.2-incubating/

kylin git commit: KYLIN-1239 attribute each htable with team contact and owner name

2015-12-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 0de917283 -> 6ec2adbbe


KYLIN-1239 attribute each htable with team contact and owner name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ec2adbb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ec2adbb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ec2adbb

Branch: refs/heads/2.0-rc
Commit: 6ec2adbbe3f26fb78c18b26f61a9c7f587ed34c0
Parents: 0de9172
Author: honma 
Authored: Thu Dec 17 11:31:55 2015 +0800
Committer: honma 
Committed: Tue Dec 22 16:10:36 2015 +0800

--
 build/conf/kylin.properties  |  5 +
 .../org/apache/kylin/common/KylinConfig.java |  6 ++
 .../realization/IRealizationConstants.java   |  5 -
 .../apache/kylin/engine/spark/SparkCubing.java   |  3 +--
 .../test_case_data/localmeta/kylin.properties|  5 +
 examples/test_case_data/sandbox/kylin.properties |  5 +
 .../storage/hbase/steps/CreateHTableJob.java |  4 +---
 .../storage/hbase/steps/CubeHTableUtil.java  | 19 ---
 .../hbase/steps/HBaseStreamingOutput.java|  2 +-
 .../kylin/storage/hbase/util/CleanHtableCLI.java |  6 +-
 10 files changed, 49 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 2c34e92..f2170bd 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -17,6 +17,11 @@
 
 ## Config for Kylin Engine ##
 
+
+# optional information for the owner of kylin platform, it can be your team's 
email
+# currently it will be attached to each kylin's htable attribute
+kylin.owner=who...@kylin.apache.org
+
 # List of web servers in use, this enables one web server instance to sync up 
with other servers.
 kylin.rest.servers=localhost:7070
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index eb76534..c65ade4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -55,6 +55,8 @@ public class KylinConfig implements Serializable {
  * NOTE: These key constants should be private or even better be removed.
  *   All external access should go through public methods.
  */
+public static final String KYLIN_OWNER = "kylin.owner";
+
 public static final String KYLIN_STORAGE_URL = "kylin.storage.url";
 
 public static final String KYLIN_METADATA_URL = "kylin.metadata.url";
@@ -706,6 +708,10 @@ public class KylinConfig implements Serializable {
 return this.getOptional(HIVE_DATABASE_FOR_INTERMEDIATE_TABLE, 
"default");
 }
 
+public String getKylinOwner() {
+return this.getOptional(KYLIN_OWNER, "");
+}
+
 public void setRunAsRemoteCommand(String v) {
 properties.setProperty(KYLIN_JOB_RUN_AS_REMOTE_CMD, v);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index 7cbcdf8..138a209 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -31,7 +31,10 @@ public class IRealizationConstants {
  * which kylin server(represented by its kylin_metadata prefix) owns this 
htable
  */
 public final static String HTableTag = "KYLIN_HOST";
-
+
+public final static String HTableOwner = "OWNER";
+
+public final static String HTableUser = "USER";
 
 public final static String HTableCreationTime = "CREATION_TIME";
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec2adbb/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 01d97fd..9a2610c 100644
---

kylin git commit: fix typo

2015-12-22 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 3fc3883a4 -> 7849a47ff


fix typo


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7849a47f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7849a47f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7849a47f

Branch: refs/heads/2.x-staging
Commit: 7849a47ffbc8d7d5dac42a46b3579aa4a7275a09
Parents: 3fc3883
Author: honma 
Authored: Tue Dec 22 15:24:07 2015 +0800
Committer: honma 
Committed: Tue Dec 22 16:11:12 2015 +0800

--
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7849a47f/README.md
--
diff --git a/README.md b/README.md
index 73313f5..10bc863 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ Get Help
 The fastest way to get response from our developers is to send email to our 
mail list ,   
 and remember to subscribe our mail list via 

 
-Licnese
+License
 
 Please refer to 
[LICENSE](https://github.com/apache/incubator-kylin/blob/master/LICENSE) file.

kylin git commit: minor, remove incubating stuff

2015-12-22 Thread lidong

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 7849a47ff -> b502ff2ff


minor, remove incubating stuff


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b502ff2f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b502ff2f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b502ff2f

Branch: refs/heads/2.x-staging
Commit: b502ff2ff6808433febb3452cb99e03182be611d
Parents: 7849a47
Author: lidongsjtu 
Authored: Tue Dec 22 16:21:33 2015 +0800
Committer: lidongsjtu 
Committed: Tue Dec 22 16:21:33 2015 +0800

--
 DISCLAIMER | 12 
 LICENSE|  2 +-
 NOTICE |  2 +-
 README.md  | 10 +-
 pom.xml| 32 
 5 files changed, 23 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b502ff2f/DISCLAIMER
--
diff --git a/DISCLAIMER b/DISCLAIMER
deleted file mode 100644
index 806ea66..000
--- a/DISCLAIMER
+++ /dev/null
@@ -1,12 +0,0 @@
-Apache Kylin is an effort undergoing incubation at the Apache Software
-Foundation (ASF), sponsored by the Apache Incubator PMC.
-
-Incubation is required of all newly accepted projects until a further
-review indicates that the infrastructure, communications, and decision
-making process have stabilized in a manner consistent with other
-successful ASF projects.
-
-While incubation status is not necessarily a reflection of the
-completeness or stability of the code, it does indicate that the
-project has yet to be fully endorsed by the ASF.
-

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502ff2f/LICENSE
--
diff --git a/LICENSE b/LICENSE
index 92e09a4..3cec384 100644
--- a/LICENSE
+++ b/LICENSE
@@ -205,7 +205,7 @@
 
 
 ==
-Apache Kylin (incubating) Subcomponents:
+Apache Kylin Subcomponents:
 
 The Apache Kylin project contains subcomponents with separate copyright
 notices and license terms. Your use of the source code for the these

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502ff2f/NOTICE
--
diff --git a/NOTICE b/NOTICE
index f18fba3..cd0c1ba 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,4 +1,4 @@
-Apache Kylin (incubating)
+Apache Kylin
 Copyright 2014-2015 The Apache Software Foundation
 
 This product includes software developed at

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502ff2f/README.md
--
diff --git a/README.md b/README.md
index 10bc863..657212f 100644
--- a/README.md
+++ b/README.md
@@ -5,20 +5,20 @@ Apache Kylin
 
 Apache Kylin is an open source Distributed Analytics Engine, contributed by 
eBay Inc., provides SQL interface and multi-dimensional analysis (OLAP) on 
Hadoop supporting extremely large datasets.
 
-For more details, see the website 
[http://kylin.incubator.apache.org](http://kylin.incubator.apache.org).
+For more details, see the website 
[http://kylin.apache.org](http://kylin.apache.org).
 
 Documentation
 =
-Please refer to 
[http://kylin.incubator.apache.org/docs/](http://kylin.incubator.apache.org/docs/).
+Please refer to [http://kylin.apache.org/docs/](http://kylin.apache.org/docs/).
 
 Get Help
 
-The fastest way to get response from our developers is to send email to our 
mail list ,   
-and remember to subscribe our mail list via 

+The fastest way to get response from our developers is to send email to our 
mail list ,   
+and remember to subscribe our mail list via 
 
 License
 
-Please refer to 
[LICENSE](https://github.com/apache/incubator-kylin/blob/master/LICENSE) file.
+Please refer to [LICENSE](https://github.com/apache/kylin/blob/master/LICENSE) 
file.
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b502ff2f/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 28ab8d7..adbf3a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
 pom
 2.0-incubating-SNAPSHOT
 Kylin:HadoopOLAPEngine
-http://kylin.incubator.apache.org
+http://kylin.apache.org
 2014
 
 
@@ -134,38 +134,38 @@
 
 
 kylin-user
-user-subscr...@kylin.incubator.apache.org
-
user-unsubscr...@kylin.incubator.apache.org
-u...@kylin.incubator.apache.org
+user-subscr...@kylin.apache.org
+user-unsubscr...@kylin.apache.org
+

[32/50] kylin git commit: KYLIN-1168 Update CubeControllerTest and JobControllerTest

2015-12-22 Thread shaofengshi

KYLIN-1168 Update CubeControllerTest and JobControllerTest

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e568c46d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e568c46d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e568c46d

Branch: refs/heads/master
Commit: e568c46d9c390db40b3680dcfd7e68b6eeb0d5c6
Parents: 4c7bb8d
Author: shaofengshi 
Authored: Fri Dec 4 11:10:51 2015 +0800
Committer: shaofengshi 
Committed: Fri Dec 4 11:10:51 2015 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java| 2 --
 .../org/apache/kylin/rest/controller/CubeControllerTest.java | 4 
 .../java/org/apache/kylin/rest/controller/JobControllerTest.java | 4 
 3 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e568c46d/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 42cf206..b94ded3 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,7 +27,6 @@ import java.util.Map;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
@@ -43,7 +42,6 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
 import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.exception.NotFoundException;

http://git-wip-us.apache.org/repos/asf/kylin/blob/e568c46d/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java 
b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index bb8cb65..09ce6c5 100644
--- 
a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ 
b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -28,6 +28,7 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.rest.request.CubeRequest;
+import org.apache.kylin.rest.service.AccessService;
 import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.service.JobService;
 import org.apache.kylin.rest.service.ServiceTestBase;
@@ -52,6 +53,8 @@ public class CubeControllerTest extends ServiceTestBase {
 CubeService cubeService;
 @Autowired
 JobService jobService;
+@Autowired
+private AccessService accessService;
 
 @Before
 public void setUp() throws Exception {
@@ -59,6 +62,7 @@ public class CubeControllerTest extends ServiceTestBase {
 cubeController = new CubeController();
 cubeController.setCubeService(cubeService);
 cubeController.setJobService(jobService);
+cubeController.setAccessService(accessService);
 
 cubeDescController = new CubeDescController();
 cubeDescController.setCubeService(cubeService);

http://git-wip-us.apache.org/repos/asf/kylin/blob/e568c46d/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java 
b/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
index fccb294..24de879 100644
--- 
a/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
+++ 
b/server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
@@ -33,6 +33,7 @@ import org.apache.kylin.job.dao.ExecutableDao;
 import org.apache.kylin.job.exception.PersistentException;
 import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.request.JobListRequest;
+import org.apache.kylin.rest.service.AccessService;
 import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.service.JobService;

[14/50] kylin git commit: KYLIN-1161,add get cube api

2015-12-22 Thread shaofengshi

KYLIN-1161,add get cube api


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f3214dd8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f3214dd8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f3214dd8

Branch: refs/heads/master
Commit: f3214dd8b5a3b56014584c260673f77a67ddc64e
Parents: 01a78b1
Author: Zhong 
Authored: Tue Nov 24 13:38:15 2015 +0800
Committer: Zhong 
Committed: Tue Nov 24 13:38:42 2015 +0800

--
 .../org/apache/kylin/rest/controller/CubeController.java  | 10 ++
 webapp/app/js/services/cubes.js   |  1 +
 2 files changed, 11 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f3214dd8/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 2f87c4f..9d1dfad 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -91,6 +91,16 @@ public class CubeController extends BasicController {
 return cubeService.getCubes(cubeName, projectName, (null == limit) ? 
20 : limit, offset);
 }
 
+@RequestMapping(value = "/get_cube", method = { RequestMethod.GET })
+@ResponseBody
+public CubeInstance getCube(@RequestParam(value = "cubeName", required = 
true) String cubeName) {
+CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+if (cube == null) {
+throw new InternalErrorException("Cannot find cube " + cubeName);
+}
+return cube;
+}
+
 /**
  * Get hive SQL of the cube
  *

http://git-wip-us.apache.org/repos/asf/kylin/blob/f3214dd8/webapp/app/js/services/cubes.js
--
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 3431c9a..3dfd2c5 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -19,6 +19,7 @@
 KylinApp.factory('CubeService', ['$resource', function ($resource, config) {
   return $resource(Config.service.url + 
'cubes/:cubeId/:propName/:propValue/:action', {}, {
 list: {method: 'GET', params: {}, isArray: true},
+getCube: {method: 'GET', params: {action:'get_cube'}, isArray: false},
 getSql: {method: 'GET', params: {propName: 'segs', action: 'sql'}, 
isArray: false},
 updateNotifyList: {method: 'PUT', params: {propName: 'notify_list'}, 
isArray: false},
 cost: {method: 'PUT', params: {action: 'cost'}, isArray: false},

[27/50] kylin git commit: KYLIN-1064 Restore disabled queries in KylinQueryTest.testVerifyQuery

2015-12-22 Thread shaofengshi

KYLIN-1064 Restore disabled queries in KylinQueryTest.testVerifyQuery

Signed-off-by: Li, Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2800025
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2800025
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2800025

Branch: refs/heads/master
Commit: e2800025581477992f9ab52999c56f7ead10b198
Parents: cfe9f0a
Author: lidongsjtu 
Authored: Tue Dec 1 21:11:52 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 2 11:09:21 2015 +0800

--
 .../apache/kylin/query/relnode/OLAPLimitRel.java |  7 +--
 .../resources/query/sql_verifyCount/query04.sql  | 19 +++
 .../query/sql_verifyCount/query04.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query05.sql  | 19 +++
 .../query/sql_verifyCount/query05.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query06.sql  | 19 +++
 .../query/sql_verifyCount/query06.sql.disable| 19 ---
 .../resources/query/sql_verifyCount/query07.sql  | 19 +++
 .../query/sql_verifyCount/query07.sql.disable| 19 ---
 9 files changed, 81 insertions(+), 78 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2800025/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index caf9e98..f6084ba 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -101,8 +101,11 @@ public class OLAPLimitRel extends SingleRel implements 
OLAPRel {
 
 @Override
 public EnumerableRel implementEnumerable(List inputs) {
-return new EnumerableLimit(getCluster(), 
getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
-sole(inputs), localOffset, localFetch);
+EnumerableRel input = sole(inputs);
+if (input instanceof OLAPRel) {
+((OLAPRel) input).replaceTraitSet(EnumerableConvention.INSTANCE);
+}
+return EnumerableLimit.create(input, localOffset, localFetch);
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2800025/query/src/test/resources/query/sql_verifyCount/query04.sql
--
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql 
b/query/src/test/resources/query/sql_verifyCount/query04.sql
new file mode 100644
index 000..9d3e409
--- /dev/null
+++ b/query/src/test/resources/query/sql_verifyCount/query04.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select * from test_kylin_fact limit 100

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2800025/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
--
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable 
b/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
deleted file mode 100644
index 9d3e409..000
--- a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements.  See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership.  The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License.  You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in

[48/50] kylin git commit: KYLIN-1170 Update website and status files to TLP

2015-12-22 Thread shaofengshi

KYLIN-1170 Update website and status files to TLP


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5adddc3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5adddc3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5adddc3

Branch: refs/heads/master
Commit: d5adddc322a5e7a3b531cb9f46d4440b1e53bca3
Parents: 57b71d1
Author: shaofengshi 
Authored: Tue Dec 15 11:03:47 2015 +0800
Committer: shaofengshi 
Committed: Tue Dec 15 11:04:14 2015 +0800

--
 DISCLAIMER | 12 ---
 LICENSE|  2 +-
 NOTICE |  2 +-
 pom.xml|  2 +-
 website/README.md  |  2 +-
 website/_dev/howto_release.md  | 60 +--
 website/_docs/release_notes.md | 63 -
 website/community/index.md |  2 +-
 8 files changed, 97 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/DISCLAIMER
--
diff --git a/DISCLAIMER b/DISCLAIMER
deleted file mode 100644
index 806ea66..000
--- a/DISCLAIMER
+++ /dev/null
@@ -1,12 +0,0 @@
-Apache Kylin is an effort undergoing incubation at the Apache Software
-Foundation (ASF), sponsored by the Apache Incubator PMC.
-
-Incubation is required of all newly accepted projects until a further
-review indicates that the infrastructure, communications, and decision
-making process have stabilized in a manner consistent with other
-successful ASF projects.
-
-While incubation status is not necessarily a reflection of the
-completeness or stability of the code, it does indicate that the
-project has yet to be fully endorsed by the ASF.
-

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/LICENSE
--
diff --git a/LICENSE b/LICENSE
index 11f2fd3..e2d4f7b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -205,7 +205,7 @@
 
 
 ==
-Apache Kylin (incubating) Subcomponents:
+Apache Kylin Subcomponents:
 
 The Apache Kylin project contains subcomponents with separate copyright
 notices and license terms. Your use of the source code for the these

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/NOTICE
--
diff --git a/NOTICE b/NOTICE
index f18fba3..cd0c1ba 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,4 +1,4 @@
-Apache Kylin (incubating)
+Apache Kylin
 Copyright 2014-2015 The Apache Software Foundation
 
 This product includes software developed at

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 83ddff5..709db3d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -142,7 +142,7 @@
 
scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git
 
scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git
 https://git-wip-us.apache.org/repos/asf/kylin.git
-kylin-1.1.1-incubating
+kylin-1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/website/README.md
--
diff --git a/website/README.md b/website/README.md
index 4f0f8a2..654a7b0 100644
--- a/website/README.md
+++ b/website/README.md
@@ -18,7 +18,7 @@ limitations under the License.
 -->
 
 # Apache Kylin website  
-This directory contains the source code for the Apache Kylin (incubating) 
website:
+This directory contains the source code for the Apache Kylin website:
 [http://kyin.apache.org](http://kylin.apache.org/).
 
 ## Kylin website's structure:

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5adddc3/website/_dev/howto_release.md
--
diff --git a/website/_dev/howto_release.md b/website/_dev/howto_release.md
index b07bdf5..3c304a6 100644
--- a/website/_dev/howto_release.md
+++ b/website/_dev/howto_release.md
@@ -156,24 +156,24 @@ $ git clean -xn
 $ mvn clean
 
 # Do a dry run of the release:prepare step, which sets version numbers.
-$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z-incubating 
-DdevelopmentVersion=(X.Y.Z+1)-incubating-SNAPSHOT -Papache-release 
-Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE}" release:prepare 2>&1 | tee 
/tmp/prepare-dry.log
+$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z 
-DdevelopmentVersion=(X.Y.Z+1)-SNAPSHOT -Papache-release 
-Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE}" release:prepare 2>&1 | tee 
/tmp/prepare-dry.log
 {% endhighlight %}
 
 __Check the artifacts:__
 
 * In

[08/50] kylin git commit: variable renaming

2015-12-22 Thread shaofengshi

variable renaming


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/320c72cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/320c72cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/320c72cc

Branch: refs/heads/master
Commit: 320c72ccb048870f5b1aa55526b6ce59cf97dbeb
Parents: b9c439e
Author: honma 
Authored: Mon Nov 23 13:21:46 2015 +0800
Committer: honma 
Committed: Mon Nov 23 13:21:46 2015 +0800

--
 bin/find-hive-dependency.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/320c72cc/bin/find-hive-dependency.sh
--
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
index ee36fc3..ea2e161 100755
--- a/bin/find-hive-dependency.sh
+++ b/bin/find-hive-dependency.sh
@@ -41,9 +41,9 @@ done
 if [ -z "$HCAT_HOME" ]
 then
 echo "HCAT_HOME not found, try to find hcatalog path from hive home"
-hive_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
-if [ -d "${hive_home}/hive-hcatalog" ]; then
-  hcatalog_home=${hive_home}/hive-hcatalog
+hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
+if [ -d "${hadoop_home}/hive-hcatalog" ]; then
+  hcatalog_home=${hadoop_home}/hive-hcatalog
 else 
   echo "Couldn't locate hcatalog installation, please make sure it is 
installed and set HCAT_HOME to the path."
   exit 1

[36/50] kylin git commit: update website, add TM mark, added 2 companies on powered by page.

2015-12-22 Thread shaofengshi

update website, add TM mark, added 2 companies on powered by page.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9457552e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9457552e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9457552e

Branch: refs/heads/master
Commit: 9457552e07555a67a5b8c0389981cb5eadd59214
Parents: 95bc7df
Author: lukehan 
Authored: Fri Dec 4 22:50:44 2015 +0800
Committer: lukehan 
Committed: Fri Dec 4 22:50:44 2015 +0800

--
 website/_data/contributors.yml | 30 --
 website/community/poweredby.md |  5 +
 website/download/index.cn.md   |  2 +-
 website/download/index.md  |  2 +-
 website/index.cn.md|  7 +++
 website/index.md   |  7 +++
 6 files changed, 37 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/_data/contributors.yml
--
diff --git a/website/_data/contributors.yml b/website/_data/contributors.yml
index dfc3d9b..17b9e38 100644
--- a/website/_data/contributors.yml
+++ b/website/_data/contributors.yml
@@ -39,7 +39,7 @@
   githubId: julianhyde
   org: Hortonworks
   role: Mentor
-- name: Luke Han (é©å¿) 
+- name: Luke Han (é©å¿) (Chair)
   apacheId: lukehan
   githubId: lukehan
   org: eBay
@@ -74,12 +74,12 @@
   apacheId: xduo
   githubId: xduo
   org: Alipay
-  role: PMC
+  role: Emeritus PMC
 - name: Ankur Bansal
   apacheId: abansal
   githubId: abansal
   org: eBay
-  role: PMC
+  role: Emeritus PMC
 - name: Xu Jiang (èæ)
   apacheId: jiangxu
   githubId: jiangxuchina
@@ -94,12 +94,30 @@
   apacheId: 
   githubId: gaodayue
   org: Meituan
-  role: committer
+  role: PMC
 - name: Hua Huang (é»æ¡¦)
   apacheId: 
   githubId: superhua
   org: MiningLAMP
+  role: PMC 
+- name: Dong Li (ææ )
+  apacheId: 
+  githubId: 
+  org: eBay
+  role: committer   
+- name: Xiaoyu Wang (çæé¨)
+  apacheId: 
+  githubId: 
+  org: JD.com
   role: committer 
-
-
+- name: Dong Li (ææ )
+  apacheId: 
+  githubId: lidongsjtu
+  org: eBay
+  role: committer 
+- name: Yerui Sun (åå¶é)
+  apacheId: sunyerui
+  githubId: 
+  org: Meituan
+  role: PMC
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/community/poweredby.md
--
diff --git a/website/community/poweredby.md b/website/community/poweredby.md
index 10d1b93..27448cc 100644
--- a/website/community/poweredby.md
+++ b/website/community/poweredby.md
@@ -25,4 +25,9 @@ __Companies & Organizations__
 * DreamSoft is using Apache Kylin as Big Data Analytics Engine for their 
customers coming from Retail, Manufacturing, Finance and others.
 * [Exponential](http://www.exponential.com)
* Using Kylin as a component in it's data platform LAMBDA to power 
inventory, campaign, behavior and demand analysis for advertising. It has been 
great at handling the size and speed constraints of old gen systems. With the 
first version already in production, we are looking forward to pushing more 
data into kylin and achieve near real time updates. 
+* [Baidu Map](http://map.baidu.com/)  (_NASDAQ: BIDU_)
+   * Apache Kylin is used for analysis Baidu Map's huge amounts of data 
about users, channels and etc, it performs quite fast with its own 
pre-calculation and index technologies along with the cubes.
+* [NetEase](http://www.163.com/)  (_NASDAQ: NTES_)
+   * Apache has been introduced as an OLAP engine for the data platform at 
NetEase, powering data analysis for various products & business with great 
performance.
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/download/index.cn.md
--
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index 81afb2e..ff4cacb 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -4,7 +4,7 @@ title: ä¸è½½
 ---
 
 __ææ°åå¸(æºä»£ç )__  
-ææ°åå¸çApache Kylinå¯ä»¥ä»ASFç½ç«ä¸è½½ï¼:
+ææ°åå¸çApache Kylinâ¢å¯ä»¥ä»ASFç½ç«ä¸è½½ï¼:
 
 * [Apache Kylin 
v1.1.1-incubating](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.1.1-incubating/)
 * [åå¸æ¥å¿](http://kylin.apache.org/docs/release_notes.html)

http://git-wip-us.apache.org/repos/asf/kylin/blob/9457552e/website/download/index.md
--
diff --git a/website/download/index.md b/website/download/index.md
index e834ccf..eed1a4b 100644
--- a/website/download/index.md
+++ b/website/download/index.md
@@ -5,7 +5,7 @@ permalink: /download/index.html
 ---
 
 __Latest

[50/50] kylin git commit: release v1.2

2015-12-22 Thread shaofengshi

release v1.2


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/13e6a434
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/13e6a434
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/13e6a434

Branch: refs/heads/master
Commit: 13e6a434a8051e7823a302ecaafb4e80cd62ed16
Parents: 5723fb6 c2589ae
Author: shaofengshi 
Authored: Tue Dec 22 16:43:33 2015 +0800
Committer: shaofengshi 
Committed: Tue Dec 22 16:43:33 2015 +0800

--
 .gitignore  |   4 +
 DISCLAIMER  |  12 --
 LICENSE |  20 +-
 NOTICE  |   2 +-
 README.md   |  10 +-
 atopcalcite/pom.xml |   2 +-
 bin/check-env.sh|   0
 bin/find-hbase-dependency.sh|   0
 bin/find-hive-dependency.sh |  24 ++-
 bin/get-properties.sh   |   0
 bin/health-check.sh |   0
 bin/kylin.sh|   2 +-
 bin/metastore.sh|  11 +-
 bin/performance-monitor.sh  |   0
 bin/sample.sh   |   2 +-
 common/pom.xml  |   2 +-
 .../org/apache/kylin/common/KylinConfig.java|  22 ++-
 .../kylin/common/lock/ZookeeperJobLock.java |   2 +-
 .../common/persistence/FileResourceStore.java   |  17 +-
 .../common/persistence/HBaseConnection.java |  11 +-
 .../common/persistence/HBaseResourceStore.java  |  48 ++---
 .../kylin/common/persistence/RawResource.java   |   6 +-
 .../kylin/common/persistence/ResourceStore.java |  39 ++--
 .../kylin/common/persistence/ResourceTool.java  |  12 +-
 .../apache/kylin/common/util/DateFormat.java|   6 +-
 .../apache/kylin/common/util/HadoopUtil.java|  99 --
 .../kylin/common/util/Log4jConfigurer.java  |  55 ++
 .../common/util/AbstractKylinTestCase.java  |   5 +
 .../common/util/HBaseMetadataTestCase.java  |   7 +-
 .../common/util/HBaseMiniclusterHelper.java |  23 ++-
 conf/kylin.properties   |   2 +-
 conf/kylin_job_conf.xml |  23 +++
 cube/pom.xml|   2 +-
 .../kylin/cube/kv/AbstractRowKeyEncoder.java|  20 +-
 dictionary/pom.xml  |   2 +-
 .../apache/kylin/dict/DateStrDictionary.java|   2 +-
 .../java/org/apache/kylin/dict/Dictionary.java  |   4 +
 .../apache/kylin/dict/DictionaryGenerator.java  | 131 -
 .../apache/kylin/dict/DictionaryManager.java|   1 +
 .../kylin/dict/IDictionaryValueEnumerator.java  |  32 
 .../dict/ListDictionaryValueEnumerator.java |  50 +
 .../dict/MultipleDictionaryValueEnumerator.java |  77 
 .../kylin/dict/TableColumnValueEnumerator.java  |  75 
 .../org/apache/kylin/dict/TrieDictionary.java   |   4 +-
 .../kylin/dict/TrieDictionaryBuilder.java   |   6 +-
 .../kylin/dict/lookup/FileTableReader.java  |   2 +
 .../apache/kylin/dict/lookup/SnapshotTable.java |   3 +-
 docs/README.md  |   2 +-
 .../metadata/cube/kylin_sales_cube.json |   3 +-
 .../test_case_data/localmeta/kylin.properties   |   2 +-
 .../minicluster/b-kylin/kylin.properties|  74 ---
 .../minicluster/b-kylin/meta/kylin.properties   |  71 ---
 .../test_case_data/minicluster/kylin.properties |   4 +-
 .../test_case_data/sandbox/kylin.properties |   2 +-
 invertedindex/pom.xml   |   2 +-
 jdbc/pom.xml|   2 +-
 .../main/java/org/apache/kylin/jdbc/Driver.java |   2 +
 .../java/org/apache/kylin/jdbc/KylinClient.java | 192 +--
 .../org/apache/kylin/jdbc/KylinConnection.java  |   6 +-
 .../util/DefaultSslProtocolSocketFactory.java   |  36 ++--
 .../jdbc/util/DefaultX509TrustManager.java  |  14 +-
 .../apache/kylin/jdbc/util/Log4jConfigurer.java |  55 ++
 job/pom.xml |   2 +-
 .../apache/kylin/job/CubeMetadataUpgrade.java   |   9 +-
 .../kylin/job/common/MapReduceExecutable.java   |  18 +-
 .../org/apache/kylin/job/cube/CubingJob.java|   2 +-
 .../org/apache/kylin/job/dao/ExecutableDao.java |  12 +-
 .../kylin/job/hadoop/AbstractHadoopJob.java |  83 ++--
 .../kylin/job/hadoop/cube/BaseCuboidMapper.java |   2 +-
 .../job/hadoop/cube/MetadataCleanupJob.java |  43 -
 .../job/hadoop/invertedindex/IIBulkLoadJob.java |   2 +-
 .../hadoop/invertedindex/IICreateHFileJob.java  |   2 +-
 .../hadoop/invertedindex/IICreateHTableJob.java |   2 +-
 .../kylin/job/tools/CubeMigrationCLI.java   | 112 ---

[44/50] kylin git commit: KYLIN-1148 fix update project issue, from nichunen <nichu...@mininglamp.com>

2015-12-22 Thread shaofengshi

KYLIN-1148 fix update project issue, from nichunen 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5f3d6ce
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5f3d6ce
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5f3d6ce

Branch: refs/heads/master
Commit: d5f3d6ce82756c00028a1c48f343137617fe370b
Parents: a39fb27
Author: jian 
Authored: Tue Dec 8 21:44:06 2015 +0800
Committer: jian 
Committed: Tue Dec 8 21:44:28 2015 +0800

--
 webapp/app/js/controllers/page.js | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d5f3d6ce/webapp/app/js/controllers/page.js
--
diff --git a/webapp/app/js/controllers/page.js 
b/webapp/app/js/controllers/page.js
index 7453d48..8382032 100644
--- a/webapp/app/js/controllers/page.js
+++ b/webapp/app/js/controllers/page.js
@@ -208,7 +208,8 @@ KylinApp.controller('PageCtrl', function ($scope, $q, 
AccessService, $modal, $lo
 var projCtrl = function ($scope, $location, $modalInstance, ProjectService, 
MessageService, projects, project, SweetAlert, ProjectModel, $cookieStore, 
$route) {
   $scope.state = {
 isEdit: false,
-oldProjName: null
+oldProjName: null,
+projectIdx: -1
   };
   $scope.isEdit = false;
   $scope.proj = {name: '', description: ''};
@@ -217,6 +218,12 @@ var projCtrl = function ($scope, $location, 
$modalInstance, ProjectService, Mess
 $scope.state.isEdit = true;
 $scope.state.oldProjName = project.name;
 $scope.proj = project;
+for (var i = 0; i < projects.length; i++){
+  if (projects[i].name === $scope.state.oldProjName){
+$scope.state.projectIdx = i;
+break;
+  }
+}
   }
 
   $scope.createOrUpdate = function () {
@@ -268,6 +275,9 @@ var projCtrl = function ($scope, $location, $modalInstance, 
ProjectService, Mess
   };
 
   $scope.cancel = function () {
+if($scope.state.isEdit){
+  projects[$scope.state.projectIdx].name = $scope.state.oldProjName;
+}
 $modalInstance.dismiss('cancel');
   };

[09/50] kylin git commit: add -L option to find command to follow symbolic links

2015-12-22 Thread shaofengshi

add -L option to find command to follow symbolic links


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/26f1665e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/26f1665e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/26f1665e

Branch: refs/heads/master
Commit: 26f1665e22ddbd67966b2f7de32571845b962993
Parents: 320c72c
Author: honma 
Authored: Mon Nov 23 13:41:17 2015 +0800
Committer: honma 
Committed: Mon Nov 23 13:41:17 2015 +0800

--
 bin/find-hive-dependency.sh | 6 +++---
 bin/sample.sh   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/26f1665e/bin/find-hive-dependency.sh
--
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
index ea2e161..cb5a43e 100755
--- a/bin/find-hive-dependency.sh
+++ b/bin/find-hive-dependency.sh
@@ -40,7 +40,7 @@ done
 # in some versions of hive hcatalog is not in hive's classpath, find it 
separately
 if [ -z "$HCAT_HOME" ]
 then
-echo "HCAT_HOME not found, try to find hcatalog path from hive home"
+echo "HCAT_HOME not found, try to find hcatalog path from hadoop home"
 hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
 if [ -d "${hadoop_home}/hive-hcatalog" ]; then
   hcatalog_home=${hadoop_home}/hive-hcatalog
@@ -53,7 +53,7 @@ else
 hcatalog_home=${HCAT_HOME}
 fi
 
-hcatalog=`find ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | 
grep -m 1 -v 'Permission denied'`
+hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 
| grep -m 1 -v 'Permission denied'`
 
 if [ -z "$hcatalog" ]
 then
@@ -62,7 +62,7 @@ then
 fi
 
 
-hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' 
-printf '%p:' | sed 's/:$//'`
+hive_lib=`find -L "$(dirname $hive_exec_path)" -name '*.jar' ! -name 
'*calcite*' -printf '%p:' | sed 's/:$//'`
 hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
 echo "hive dependency: $hive_dependency"
 export hive_dependency

http://git-wip-us.apache.org/repos/asf/kylin/blob/26f1665e/bin/sample.sh
--
diff --git a/bin/sample.sh b/bin/sample.sh
index d53393c..e02ef51 100755
--- a/bin/sample.sh
+++ b/bin/sample.sh
@@ -19,7 +19,7 @@
 
 dir=$(dirname ${0})
 source ${dir}/check-env.sh
-job_jar=`find ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
+job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
 echo "Going to create sample tables in hive..."
 cd ${KYLIN_HOME}/sample_cube/data
 hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }

[49/50] kylin git commit: [maven-release-plugin] prepare release kylin-1.2

2015-12-22 Thread shaofengshi

[maven-release-plugin] prepare release kylin-1.2


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2589aee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2589aee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2589aee

Branch: refs/heads/master
Commit: c2589aee4ac5537b460b3b02fa89cdb3a922d64e
Parents: d5adddc
Author: shaofengshi 
Authored: Tue Dec 15 13:50:03 2015 +0800
Committer: shaofengshi 
Committed: Tue Dec 15 13:50:03 2015 +0800

--
 atopcalcite/pom.xml   | 2 +-
 common/pom.xml| 2 +-
 cube/pom.xml  | 2 +-
 dictionary/pom.xml| 2 +-
 invertedindex/pom.xml | 2 +-
 jdbc/pom.xml  | 2 +-
 job/pom.xml   | 2 +-
 metadata/pom.xml  | 2 +-
 monitor/pom.xml   | 2 +-
 pom.xml   | 2 +-
 query/pom.xml | 2 +-
 server/pom.xml| 2 +-
 storage/pom.xml   | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1d9a3e1..2833b71 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 035de00..1e7d5b2 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/cube/pom.xml
--
diff --git a/cube/pom.xml b/cube/pom.xml
index 9519be5..7c4038f 100644
--- a/cube/pom.xml
+++ b/cube/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/dictionary/pom.xml
--
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
index 4a682e4..f0b2d13 100644
--- a/dictionary/pom.xml
+++ b/dictionary/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/invertedindex/pom.xml
--
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index 7647af2..c8d72b8 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
 
 kylin
 org.apache.kylin
-1.2-SNAPSHOT
+1.2
 
 4.0.0
 kylin-invertedindex

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 132c68c..fc76490 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@

org.apache.kylin
kylin
-   1.2-SNAPSHOT
+   1.2

 


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/job/pom.xml
--
diff --git a/job/pom.xml b/job/pom.xml
index fb677b3..23085d0 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 kylin-job

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/metadata/pom.xml
--
diff --git a/metadata/pom.xml b/metadata/pom.xml
index 259962b..5ea8b13 100644
--- a/metadata/pom.xml
+++ b/metadata/pom.xml
@@ -27,7 +27,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/monitor/pom.xml
--
diff --git a/monitor/pom.xml b/monitor/pom.xml
index cb2df81..b67969d 100644
--- a/monitor/pom.xml
+++ b/monitor/pom.xml
@@ -22,7 +22,7 @@
 
 org.apache.kylin
 kylin
-1.2-SNAPSHOT
+1.2
 
 
 4.0.0

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2589aee/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 709db3d..b1c81e2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
 org.apache.kylin
 kylin
 pom
-

[31/50] kylin git commit: only cache ScanOutOfLimitException exceptions

2015-12-22 Thread shaofengshi

only cache ScanOutOfLimitException exceptions


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c7bb8de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c7bb8de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c7bb8de

Branch: refs/heads/master
Commit: 4c7bb8de94d4e910674366cf2922545443c1288c
Parents: 9e069ee
Author: honma 
Authored: Thu Dec 3 17:48:59 2015 +0800
Committer: honma 
Committed: Thu Dec 3 17:49:32 2015 +0800

--
 .../org/apache/kylin/rest/controller/QueryController.java   | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c7bb8de/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 5ac8296..c7499bd 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -44,6 +44,7 @@ import org.apache.kylin.rest.request.SaveSqlRequest;
 import org.apache.kylin.rest.response.SQLResponse;
 import org.apache.kylin.rest.service.QueryService;
 import org.apache.kylin.rest.util.QueryUtil;
+import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -191,18 +192,18 @@ public class QueryController extends BasicController {
 
 sqlResponse = new SQLResponse(null, null, 0, true, errMsg);
 
-// Access exception is bind with each user, it will not be cached
-if ((e instanceof AccessDeniedException) == false) {
+// for exception queries, only cache ScanOutOfLimitException
+if (e instanceof ScanOutOfLimitException) {
 Cache exceptionCache = 
cacheManager.getCache(EXCEPTION_QUERY_CACHE);
 exceptionCache.put(new Element(sqlRequest, sqlResponse));
 }
 }
 
 queryService.logQuery(sqlRequest, sqlResponse);
-
+
 if (sqlResponse.getIsException())
 throw new 
InternalErrorException(sqlResponse.getExceptionMessage());
-
+
 return sqlResponse;
 }

[38/50] kylin git commit: KYLIN-1205 Make HBase scan parameters configurable and allow reset of HConnection

2015-12-22 Thread shaofengshi

KYLIN-1205 Make HBase scan parameters configurable and allow reset of 
HConnection


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0c5fd3b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0c5fd3b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0c5fd3b

Branch: refs/heads/master
Commit: b0c5fd3bc7cce69dd5f0413d172fd7c7a79d9f5a
Parents: e85b243
Author: Li, Yang 
Authored: Tue Dec 8 10:43:03 2015 +0800
Committer: Li, Yang 
Committed: Tue Dec 8 10:43:03 2015 +0800

--
 .../org/apache/kylin/common/KylinConfig.java |  8 
 .../common/persistence/HBaseConnection.java  |  4 
 .../apache/kylin/rest/service/CacheService.java  |  2 ++
 .../storage/hbase/CubeSegmentTupleIterator.java  | 19 +++
 4 files changed, 29 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 9645b81..7816487 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -499,7 +499,15 @@ public class KylinConfig {
 public int getHBaseKeyValueSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", 
"10485760"));
 }
+
+public int getHBaseScanCacheRows() {
+return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
+}
 
+public int getHBaseScanMaxResultSize() {
+return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
+}
+
 public String getHbaseDefaultCompressionCodec() {
 return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC, "");
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
--
diff --git 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
index dcefc24..9c86376 100644
--- 
a/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
+++ 
b/common/src/main/java/org/apache/kylin/common/persistence/HBaseConnection.java
@@ -58,6 +58,10 @@ public class HBaseConnection {
 }
 });
 }
+
+public static void clearCache() {
+ConnPool.clear();
+}
 
 public static HConnection get(String url) {
 // find configuration

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 38d7400..2328592 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -21,6 +21,7 @@ package org.apache.kylin.rest.service;
 import java.io.IOException;
 import java.util.List;
 
+import org.apache.kylin.common.persistence.HBaseConnection;
 import org.apache.kylin.common.restclient.Broadcaster;
 import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.CubeManager;
@@ -101,6 +102,7 @@ public class CacheService extends BasicService {
 RealizationRegistry.clearCache();
 ProjectManager.clearCache();
 BasicService.resetOLAPDataSources();
+HBaseConnection.clearCache();
 break;
 default:
 throw new RuntimeException("invalid cacheType:" + cacheType);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0c5fd3b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
--
diff --git 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
index 9efbb79..fa4ccd7 100644
--- 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
+++ 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterList;

[45/50] kylin git commit: KYLIN-1215 minor, update website member's info on community page

2015-12-22 Thread shaofengshi

KYLIN-1215 minor, update website member's info on community page


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8d7d43d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8d7d43d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8d7d43d

Branch: refs/heads/master
Commit: b8d7d43d6e4969a0b856f9eaf33c7c499c9747f8
Parents: d5f3d6c
Author: Xiaoyu Wang 
Authored: Wed Dec 9 13:07:10 2015 +0800
Committer: Xiaoyu Wang 
Committed: Wed Dec 9 13:07:10 2015 +0800

--
 website/_data/contributors.yml | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b8d7d43d/website/_data/contributors.yml
--
diff --git a/website/_data/contributors.yml b/website/_data/contributors.yml
index 41145f3..d32b93f 100644
--- a/website/_data/contributors.yml
+++ b/website/_data/contributors.yml
@@ -91,28 +91,28 @@
   org: eBay
   role: PMC
 - name: Dayue Gao (é«å¤§æ)
-  apacheId: 
+  apacheId: gaodayue
   githubId: gaodayue
   org: Meituan
   role: PMC
 - name: Hua Huang (é»æ¡¦)
-  apacheId: 
+  apacheId: hhuang
   githubId: superhua
   org: MiningLAMP
-  role: PMC 
+  role: PMC
 - name: Dong Li (ææ )
-  apacheId: lidong 
+  apacheId: lidong
   githubId: lidongsjtu
   org: eBay
-  role: committer   
+  role: committer
 - name: Xiaoyu Wang (çæé¨)
   apacheId: wangxiaoyu
-  githubId: 
+  githubId: xiaowangyu
   org: JD.com
-  role: committer 
+  role: committer
 - name: Yerui Sun (åå¶é)
   apacheId: sunyerui
-  githubId: 
+  githubId: sunyerui
   org: Meituan
   role: PMC

[15/50] kylin git commit: KYLIN-1157 copy ACL when migrate cube - small fix

2015-12-22 Thread shaofengshi

KYLIN-1157 copy ACL when migrate cube - small fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a65df19
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a65df19
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a65df19

Branch: refs/heads/master
Commit: 9a65df1940cc1fea05ef146da275d07919e25fad
Parents: 7e7edff
Author: shaofengshi 
Authored: Tue Nov 24 13:39:13 2015 +0800
Committer: shaofengshi 
Committed: Tue Nov 24 13:41:01 2015 +0800

--
 .../main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a65df19/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
--
diff --git a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java 
b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
index e3f7c11..ca756d1 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
@@ -76,7 +76,7 @@ public class CubeMigrationCLI {
 
 public static void main(String[] args) throws IOException, 
InterruptedException {
 
-if (args.length != 6) {
+if (args.length != 7) {
 usage();
 System.exit(1);
 }
@@ -123,7 +123,7 @@ public class CubeMigrationCLI {
 renameFoldersInHdfs(cube);
 changeHtableHost(cube);
 addCubeIntoProject(cubeName, projectName);
-if (Boolean.valueOf(copyAcl).booleanValue() == true) {
+if (Boolean.parseBoolean(copyAcl) == true) {
 copyACL(cube);
 }

[13/50] kylin git commit: KYLIN-1162 be compatible with YARN-2605

2015-12-22 Thread shaofengshi

KYLIN-1162 be compatible with YARN-2605


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01a78b1a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01a78b1a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01a78b1a

Branch: refs/heads/master
Commit: 01a78b1ab107d4b3621086862811b909ad020f62
Parents: 2a9c0cc
Author: shaofengshi 
Authored: Tue Nov 24 10:39:18 2015 +0800
Committer: shaofengshi 
Committed: Tue Nov 24 10:39:51 2015 +0800

--
 .../kylin/job/tools/HadoopStatusGetter.java | 27 +---
 1 file changed, 18 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/01a78b1a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java 
b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
index 0cd4d43..e2c0aa0 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
@@ -80,16 +80,25 @@ public class HadoopStatusGetter {
 client.executeMethod(get);
 
 String redirect = null;
-Header h = get.getResponseHeader("Refresh");
+Header h = get.getResponseHeader("Location");
 if (h != null) {
-String s = h.getValue();
-int cut = s.indexOf("url=");
-if (cut >= 0) {
-redirect = s.substring(cut + 4);
-
-if (isValidURL(redirect) == false) {
-log.info("Get invalid redirect url, skip it: " + 
redirect);
-continue;
+redirect = h.getValue();
+if (isValidURL(redirect) == false) {
+log.info("Get invalid redirect url, skip it: " + 
redirect);
+continue;
+}
+} else {
+h = get.getResponseHeader("Refresh");
+if (h != null) {
+String s = h.getValue();
+int cut = s.indexOf("url=");
+if (cut >= 0) {
+redirect = s.substring(cut + 4);
+
+if (isValidURL(redirect) == false) {
+log.info("Get invalid redirect url, skip it: " 
+ redirect);
+continue;
+}
 }
 }
 }

[23/50] kylin git commit: KYLIN-1168 fix "Update data model is not allowed! Please create a new cube if needed" error

2015-12-22 Thread shaofengshi

KYLIN-1168 fix "Update data model is not allowed! Please create a new cube if 
needed" error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/04c6607f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/04c6607f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/04c6607f

Branch: refs/heads/master
Commit: 04c6607fee3775c7225d19ff55503e38238cecdb
Parents: 55c448b
Author: shaofengshi 
Authored: Wed Nov 25 17:25:14 2015 +0800
Committer: shaofengshi 
Committed: Wed Nov 25 17:25:44 2015 +0800

--
 .../kylin/metadata/model/DataModelDesc.java | 38 ---
 .../kylin/rest/controller/CubeController.java   | 39 +---
 .../apache/kylin/rest/service/CacheService.java |  8 ++--
 .../apache/kylin/rest/service/CubeService.java  |  2 +-
 4 files changed, 31 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/04c6607f/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java 
b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 88fafdf..f8dc54d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -220,44 +220,6 @@ public class DataModelDesc extends RootPersistentEntity {
 }
 }
 
-/**
- * Check whether two data model are compatible or not. Compatible means
- * having the same structure. Tow models could be compatible even they
- * have different UUID or last modified time.
- * @param that model to compare with
- * @return true if compatible, false otherwise.
- */
-public boolean compatibleWith(DataModelDesc that) {
-if (this == that)
-return true;
-
-if (that == null)
-return false;
-
-try {
-String thisRepr = excludeHeaderInfo(this);
-String thatRepr = excludeHeaderInfo(that);
-return StringUtils.equals(thisRepr, thatRepr);
-
-} catch (IOException e) {
-logger.error("Failed to serialize DataModelDesc to string", e);
-return false;
-}
-}
-
-private String excludeHeaderInfo(DataModelDesc modelDesc) throws 
IOException {
-// make a copy
-String repr = JsonUtil.writeValueAsString(modelDesc);
-DataModelDesc copy = JsonUtil.readValue(repr, DataModelDesc.class);
-
-copy.setUuid(null);
-copy.setLastModified(0);
-copy.setCapacity(RealizationCapacity.MEDIUM);
-copy.setFilterCondition(null);
-copy.setPartitionDesc(null);
-return JsonUtil.writeValueAsString(copy);
-}
-
 @Override
 public String toString() {
 return "DataModelDesc [name=" + name + "]";

http://git-wip-us.apache.org/repos/asf/kylin/blob/04c6607f/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 9d1dfad..8565d5e 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -359,8 +359,8 @@ public class CubeController extends BasicController {
 }
 
 MetadataManager metadataManager = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
-// KYLIN-958: disallow data model structure change
 DataModelDesc modelDesc = null;
+DataModelDesc oldModelDesc = null;
 if (StringUtils.isNotEmpty(cubeRequest.getModelDescData())) {
 modelDesc = deserializeDataModelDesc(cubeRequest);
 if (modelDesc == null) {
@@ -373,15 +373,11 @@ public class CubeController extends BasicController {
 return errorRequest(cubeRequest, "CubeDesc.model_name " + 
desc.getModelName() + " not consistent with model " + modeName);
 }
 
-DataModelDesc oldModelDesc = 
metadataManager.getDataModelDesc(modeName);
+oldModelDesc = metadataManager.getDataModelDesc(modeName);
 if (oldModelDesc == null) {
 return errorRequest(cubeRequest, "Data model " + modeName + " 
not found");
 }
 
-if (!modelDesc.compatibleWith(oldModelDesc)) {
-return errorRequest(cubeRequest, "Update data model is not 
allowed! Please create a new cube if needed");
-}
-

[30/50] kylin git commit: KYLIN-1168 add UT

2015-12-22 Thread shaofengshi

KYLIN-1168 add UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e069eed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e069eed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e069eed

Branch: refs/heads/master
Commit: 9e069eed066f356909ac34a570707219d95ecb5e
Parents: 316bc04
Author: shaofengshi 
Authored: Thu Dec 3 14:54:22 2015 +0800
Committer: shaofengshi 
Committed: Thu Dec 3 14:57:19 2015 +0800

--
 .../kylin/rest/controller/CubeController.java   |  50 +++--
 .../kylin/rest/controller/JobController.java|   2 +-
 .../apache/kylin/rest/service/BasicService.java |  10 -
 .../apache/kylin/rest/service/CacheService.java |  24 ++-
 .../apache/kylin/rest/service/CubeService.java  |  42 +
 .../kylin/rest/service/CacheServiceTest.java| 186 +++
 .../kylin/rest/service/CubeServiceTest.java |   4 -
 .../kylin/rest/service/JobServiceTest.java  |   1 -
 .../kylin/rest/service/QueryServiceTest.java|   1 -
 9 files changed, 219 insertions(+), 101 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9e069eed/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f27f312..42cf206 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -41,6 +41,8 @@ import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
 import org.apache.kylin.rest.exception.InternalErrorException;
@@ -49,6 +51,8 @@ import org.apache.kylin.rest.request.CubeRequest;
 import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.response.GeneralResponse;
 import org.apache.kylin.rest.response.HBaseResponse;
+import org.apache.kylin.rest.security.AclPermission;
+import org.apache.kylin.rest.service.AccessService;
 import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.service.JobService;
 import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
@@ -83,6 +87,9 @@ public class CubeController extends BasicController {
 private CubeService cubeService;
 
 @Autowired
+private AccessService accessService;
+
+@Autowired
 private JobService jobService;
 
 @RequestMapping(value = "", method = { RequestMethod.GET })
@@ -278,6 +285,7 @@ public class CubeController extends BasicController {
 
 try {
 cubeService.deleteCube(cube);
+accessService.clean(cube, true);
 } catch (Exception e) {
 logger.error(e.getLocalizedMessage(), e);
 throw new InternalErrorException("Failed to delete cube. " + " 
Caused by: " + e.getMessage(), e);
@@ -285,21 +293,25 @@ public class CubeController extends BasicController {
 }
 
 /**
- * Get available table list of the input database
+ * Create a new Cube
  *
- * @return Table metadata array
+ * @return cubeRequest cube change request
  * @throws IOException
  */
 @RequestMapping(value = "", method = { RequestMethod.POST })
 @ResponseBody
 public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) {
 //Update Model
-MetadataManager metaManager = 
MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+MetadataManager metaManager = 
MetadataManager.getInstance(cubeService.getConfig());
 DataModelDesc modelDesc = deserializeDataModelDesc(cubeRequest);
-if (modelDesc == null || StringUtils.isEmpty(modelDesc.getName())) {
-return cubeRequest;
+if (modelDesc == null) {
+return errorRequest(cubeRequest, "Missing ModelDesc data in the 
request.");
 }
 
+if (StringUtils.isEmpty(modelDesc.getName())) {
+return errorRequest(cubeRequest, "Missing modelName.");
+}
+
 try {
 DataModelDesc existingModel = 
metaManager.getDataModelDesc(modelDesc.getName());
 if (existingModel == null) {
@@ -309,7 +321,6 @@ public class CubeController extends BasicController {

[39/50] kylin git commit: KYLIN-1128 clone cube metadata

2015-12-22 Thread shaofengshi

KYLIN-1128 clone cube metadata


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ad77cca
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ad77cca
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ad77cca

Branch: refs/heads/master
Commit: 9ad77cca832dfabb562b8e12d5e28e9e505e245d
Parents: d18b73a
Author: jian 
Authored: Fri Dec 4 12:48:38 2015 +0800
Committer: jian 
Committed: Tue Dec 8 11:03:05 2015 +0800

--
 .../apache/kylin/metadata/MetadataManager.java  | 21 
 .../kylin/rest/controller/CubeController.java   | 15 ++
 webapp/app/js/controllers/cubes.js  |  4 +---
 webapp/app/partials/cubes/cube_clone.html   | 13 
 4 files changed, 33 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9ad77cca/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
--
diff --git 
a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java 
b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index b540588..800ea44 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -40,6 +40,7 @@ import 
org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.ProjectManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -341,6 +342,26 @@ public class MetadataManager {
 return saveDataModelDesc(desc);
 }
 
+// sync on update
+public DataModelDesc dropModel(DataModelDesc desc) throws IOException {
+logger.info("Dropping model '" + desc.getName() + "'");
+ResourceStore store = getStore();
+if (desc != null)
+store.deleteResource(desc.getResourcePath());
+// clean model cache
+this.afterModelDropped(desc);
+return desc;
+}
+
+private void afterModelDropped(DataModelDesc desc) {
+removeModelCache(desc);
+}
+
+private void removeModelCache(DataModelDesc desc) {
+dataModelDescMap.remove(desc.getName());
+}
+
+
 private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) 
throws IOException {
 dataModelDesc.init(this.getAllTablesMap());
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9ad77cca/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b35a35c..95169da 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeSegment;
@@ -260,7 +261,6 @@ public class CubeController extends BasicController {
 @ResponseBody
 public CubeInstance cloneCube(@PathVariable String cubeName,@RequestBody 
CubeRequest cubeRequest) {
 String targetCubeName = cubeRequest.getCubeName();
-String targetModelName = cubeRequest.getModelDescData();
 String project = cubeRequest.getProject();
 
 CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -275,11 +275,13 @@ public class CubeController extends BasicController {
 
 DataModelDesc modelDesc = metaManager.getDataModelDesc(modelName);
 
-modelDesc.setName(targetModelName);
+//model name same as cube
+modelDesc.setName(targetCubeName);
 modelDesc.setLastModified(0);
 modelDesc.setUuid(UUID.randomUUID().toString());
+DataModelDesc newModel = null;
 try {
-metaManager.createDataModelDesc(modelDesc);
+newModel = metaManager.createDataModelDesc(modelDesc);
 } catch (IOException e) {
 throw new InternalErrorException("failed to clone 
DataModelDesc",e);
 }
@@ -287,11 +289,16 @@ public class CubeController extends BasicController {
 cubeDesc.setName(targetCubeName);
 cubeDesc.setLastModified(0);
 cubeDesc.setUuid(UUID.randomUUID().toString());
-

[24/50] kylin git commit: KYLIN-1168 fix UT

2015-12-22 Thread shaofengshi

KYLIN-1168 fix UT

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f5b1561
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f5b1561
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f5b1561

Branch: refs/heads/master
Commit: 0f5b1561712b290a00c100e00faf97f430742698
Parents: 04c6607
Author: shaofengshi 
Authored: Thu Nov 26 11:27:56 2015 +0800
Committer: shaofengshi 
Committed: Thu Nov 26 11:27:56 2015 +0800

--
 .../org/apache/kylin/rest/controller/CubeController.java |  1 +
 .../java/org/apache/kylin/rest/service/CacheService.java |  7 +--
 .../org/apache/kylin/rest/service/CacheServiceTest.java  | 11 ---
 3 files changed, 10 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5b1561/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8565d5e..f27f312 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -414,6 +414,7 @@ public class CubeController extends BasicController {
 if (updateModelSuccess == true && updateCubeSuccess == false ) {
 // recover data model
 try {
+oldModelDesc.setLastModified(modelDesc.getLastModified());
 metadataManager.updateDataModelDesc(oldModelDesc);
 } catch (IOException e) {
 logger.error("Failed to recover data model desc:" + 
e.getLocalizedMessage(), e);

http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5b1561/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index c953b7e..7a3238c 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -59,8 +59,10 @@ public class CacheService extends BasicService {
 cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
 break;
 case CUBE_DESC:
-String modelName = 
getCubeDescManager().getCubeDesc(cacheKey).getModelName();
-getMetadataManager().reloadDataModelDesc(modelName);
+if (getCubeDescManager().getCubeDesc(cacheKey) != null) {
+String modelName = 
getCubeDescManager().getCubeDesc(cacheKey).getModelName();
+getMetadataManager().reloadDataModelDesc(modelName);
+}
 getCubeDescManager().reloadCubeDesc(cacheKey);
 IIDescManager.clearCache();
 CubeDescManager.clearCache();
@@ -143,4 +145,5 @@ public class CacheService extends BasicService {
 throw new RuntimeException("error " + log, e);
 }
 }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0f5b1561/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java 
b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 0436dc9..44755d4 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -47,17 +47,13 @@ import 
org.apache.kylin.rest.broadcaster.BroadcasterReceiveServlet;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
 import org.slf4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
 
 /**
- * Created by qianzhou on 1/16/15.
  */
-
+@Ignore ("Not working for some time")
 public class CacheServiceTest extends LocalFileMetadataTestCase {
 
 private static Server server;
@@ -281,6 +277,7 @@ public class CacheServiceTest extends 
LocalFileMetadataTestCase {
 }
 
 @Test
+//@Ignore ("disable this as only data model change will not trigger a 
broad cast event, see KYLIN-1168")
 public void testMetaCRUD() throws

[28/50] kylin git commit: KYLIN-1190 Make memory budget per query configurable

2015-12-22 Thread shaofengshi

KYLIN-1190 Make memory budget per query configurable

Signed-off-by: Li, Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7888c87f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7888c87f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7888c87f

Branch: refs/heads/master
Commit: 7888c87fa190e950227eee80d59f8a108b93a18f
Parents: e280002
Author: lidongsjtu 
Authored: Tue Dec 1 16:32:17 2015 +0800
Committer: Li, Yang 
Committed: Wed Dec 2 11:11:59 2015 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfig.java|  4 
 .../org/apache/kylin/storage/hbase/CubeStorageEngine.java | 10 +++---
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7888c87f/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index a8f7c5d..9645b81 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -492,6 +492,10 @@ public class KylinConfig {
 return 
Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true"));
 }
 
+public long getQueryMemBudget() {
+return Long.parseLong(this.getOptional("kylin.query.mem.budget", 
String.valueOf(3L * 1024 * 1024 * 1024)));
+}
+
 public int getHBaseKeyValueSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", 
"10485760"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7888c87f/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
--
diff --git 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index ed12781..bbf088e 100644
--- 
a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ 
b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -76,7 +76,6 @@ public class CubeStorageEngine implements IStorageEngine {
 private static final Logger logger = 
LoggerFactory.getLogger(CubeStorageEngine.class);
 
 private static final int MERGE_KEYRANGE_THRESHOLD = 100;
-private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; 
// 3G
 
 private final CubeInstance cubeInstance;
 private final CubeDesc cubeDesc;
@@ -643,8 +642,13 @@ public class CubeStorageEngine implements IStorageEngine {
 }
 }
 
-long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
-context.setThreshold((int) rowEst);
+long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / 
rowSizeEst;
+if (rowEst > 0) {
+logger.info("Memory budget is set to: " + rowEst);
+context.setThreshold((int) rowEst);
+} else {
+logger.info("Memory budget is not set.");
+}
 }
 
 private void setLimit(TupleFilter filter, StorageContext context) {

svn commit: r1721410 - in /kylin/site: cn/download/index.html community/index.html development/howto_package.html development/howto_release.html download/index.html feed.xml

2015-12-22 Thread shaofengshi

Author: shaofengshi
Date: Tue Dec 22 14:20:24 2015
New Revision: 1721410

URL: http://svn.apache.org/viewvc?rev=1721410=rev
Log:
update with v1.2 release

Modified:
kylin/site/cn/download/index.html
kylin/site/community/index.html
kylin/site/development/howto_package.html
kylin/site/development/howto_release.html
kylin/site/download/index.html
kylin/site/feed.xml

Modified: kylin/site/cn/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/cn/download/index.html?rev=1721410=1721409=1721410=diff
==
--- kylin/site/cn/download/index.html (original)
+++ kylin/site/cn/download/index.html Tue Dec 22 14:20:24 2015
@@ -166,17 +166,17 @@
 ææ°åå¸çApache Kylinâ¢å¯ä»¥ä»ASFç½ç«ä¸è½½ï¼:
 
 
-  http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.1.1-incubating/;>Apache
 Kylin v1.1.1-incubating
+  http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-1.2/;>Apache 
Kylin v1.2
   http://kylin.apache.org/docs/release_notes.html;>åå¸æ¥å¿
-  Git æ ç¾: https://github.com/apache/kylin/tree/kylin-1.1.1-incubating;>kylin-1.1.1-incubating
-  Git Commit: https://github.com/apache/kylin/commit/6a9499b4d0fabb54211a8a536c2e18d3fe8b4a5d;>6a9499b4d0fabb54211a8a536c2e18d3fe8b4a5d
+  Git æ ç¾: https://github.com/apache/kylin/tree/kylin-1.2;>kylin-1.2
+  Git Commit: https://github.com/apache/kylin/commit/c2589aee4ac5537b460b3b02fa89cdb3a922d64e;>c2589aee4ac5537b460b3b02fa89cdb3a922d64e
 
 
 äºè¿å¶å for HBase 0.98/0.99
 ä¸ºæ¹ä¾¿ä½¿ç¨ï¼æä»¬æä¾é¢æåçäºè¿å¶å®è£åï¼
 
 
-  https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.1.1-incubating/apache-kylin-1.1.1-incubating-bin.tar.gz;>apache-kylin-1.1.1-incubating-bin.tar.gz
+  https://dist.apache.org/repos/dist/release/kylin/apache-kylin-1.2/apache-kylin-1.2-bin.tar.gz;>apache-kylin-1.2-bin.tar.gz
   http://kylin.apache.org/docs/install;>å®è£å¸®å©
 
 
@@ -204,7 +204,6 @@ Kylin ODBC é©±å¨å¯ä»¥ä»�
 
 
   http://kylin.apache.org/download/KylinODBCDriver.zip;>Kylin 
ODBC é©±å¨
-  http://kylin.apache.org/download/KylinODBCDriver.zip;>Kylin 
ODBC é©±å¨
 
 
 

Modified: kylin/site/community/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/community/index.html?rev=1721410=1721409=1721410=diff
==
--- kylin/site/community/index.html (original)
+++ kylin/site/community/index.html Tue Dec 22 14:20:24 2015
@@ -213,7 +213,7 @@
 For convenience, thereâs a forum style mailing list archives which not 
part of offical Apache archives:
 
 
-  http://apache-kylin-incubating.74782.x6.nabble.com;>Developer 
List archive on Nabble
+  http://apache-kylin.74782.x6.nabble.com;>Developer List archive 
on Nabble
 
 
 Social Media

Modified: kylin/site/development/howto_package.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_package.html?rev=1721410=1721409=1721410=diff
==
--- kylin/site/development/howto_package.html (original)
+++ kylin/site/development/howto_package.html Tue Dec 22 14:20:24 2015
@@ -447,6 +447,11 @@
 
 In order to generate binary package, maven and 
npm are pre-requisites.
 
+If youâre behind a proxy server, both npm and bower need be told with the 
proxy info before running ./script/package.sh:
+
+export http_proxy=http://your-proxy-host:port
+npm config set proxy 
http://your-proxy-host:port
+




Modified: kylin/site/development/howto_release.html
URL: 
http://svn.apache.org/viewvc/kylin/site/development/howto_release.html?rev=1721410=1721409=1721410=diff
==
--- kylin/site/development/howto_release.html (original)
+++ kylin/site/development/howto_release.html Tue Dec 22 14:20:24 2015
@@ -591,28 +591,28 @@ If youâre the first time to do rele
 $ mvn clean
 
 # Do a dry run of the release:prepare step, which sets version 
numbers.
-$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z-incubating -DdevelopmentVersion=(X.Y.Z+1)-incubating-SNAPSHOT 
-Papache-release -Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE}" release:prepare 21 | tee 
/tmp/prepare-dry.log
+$ mvn -DdryRun=true -DskipTests -DreleaseVersion=X.Y.Z -DdevelopmentVersion=(X.Y.Z+1)-SNAPSHOT -Papache-release 
-Darguments="-Dgpg.passphrase=${GPG_PASSPHRASE}" release:prepare 21 | tee 
/tmp/prepare-dry.log
 
 Check the artifacts:
 
 
   In the target directory should be 
these 8 files, among others:
 
-  apache-kylin-X.Y.Z-incubating-SNAPSHOT-src.tar.gz
-  apache-kylin-X.Y.Z-incubating-SNAPSHOT-src.tar.gz.asc
-  apache-kylin-X.Y.Z-incubating-SNAPSHOT-src.tar.gz.md5
-  apache-kylin-X.Y.Z-incubating-SNAPSHOT-src.tar.gz.sha1
-

kylin git commit: KYLIN-1246 get cubes API update - offset, limit not required

2015-12-22 Thread zhongjian

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 430319316 -> ef4c524fc


KYLIN-1246 get cubes API update - offset,limit not required


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ef4c524f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ef4c524f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ef4c524f

Branch: refs/heads/1.x-staging
Commit: ef4c524fc1e4168bb1ca6d4e6e4e6cec57e56763
Parents: 4303193
Author: jian 
Authored: Wed Dec 23 15:52:39 2015 +0800
Committer: jian 
Committed: Wed Dec 23 15:52:58 2015 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java  | 4 ++--
 .../main/java/org/apache/kylin/rest/service/CubeService.java   | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ef4c524f/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java 
b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 95169da..679e4e6 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -93,8 +93,8 @@ public class CubeController extends BasicController {
 
 @RequestMapping(value = "", method = { RequestMethod.GET })
 @ResponseBody
-public List getCubes(@RequestParam(value = "cubeName", 
required = false) String cubeName, @RequestParam(value = "projectName", 
required = false) String projectName, @RequestParam("limit") Integer limit, 
@RequestParam("offset") Integer offset) {
-return cubeService.getCubes(cubeName, projectName, (null == limit) ? 
20 : limit, offset);
+public List getCubes(@RequestParam(value = "cubeName", 
required = false) String cubeName, @RequestParam(value = "projectName", 
required = false) String projectName, @RequestParam(value = "limit", required = 
false) Integer limit, @RequestParam(value = "offset", required = false) Integer 
offset) {
+return cubeService.getCubes(cubeName, projectName, limit, offset);
 }
 
 @RequestMapping(value = "/{cubeName}", method = { RequestMethod.GET })

http://git-wip-us.apache.org/repos/asf/kylin/blob/ef4c524f/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java 
b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index af518ef..b8de4d4 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -110,12 +110,12 @@ public class CubeService extends BasicService {
 }
 
 public List getCubes(final String cubeName, final String 
projectName, final Integer limit, final Integer offset) {
-int climit = (null == limit) ? 30 : limit;
-int coffset = (null == offset) ? 0 : offset;
-
 List cubes;
 cubes = listAllCubes(cubeName, projectName);
 
+int coffset = (null == offset) ? 0 : offset;
+int climit = (null == limit) ? cubes.size() : limit;
+
 if (cubes.size() <= coffset) {
 return Collections.emptyList();
 }

60 matches

Mail list logo