kylin git commit: KYLIN-2423 Model should always include PK/FK as dimensions

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2423 [created] 0f772b352


KYLIN-2423 Model should always include PK/FK as dimensions


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f772b35
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f772b35
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f772b35

Branch: refs/heads/KYLIN-2423
Commit: 0f772b3529ef4855b5ea972eaf0f3d82a971dbd9
Parents: 7611338
Author: Li Yang 
Authored: Sat Feb 4 15:19:31 2017 +0800
Committer: Li Yang 
Committed: Sat Feb 4 15:19:31 2017 +0800

--
 .../kylin/metadata/model/DataModelDesc.java | 76 
 .../kylin/metadata/model/JoinTableDesc.java |  7 +-
 .../apache/kylin/metadata/model/JoinsTree.java  | 11 +--
 .../metadata/model/ModelDimensionDesc.java  |  2 +
 4 files changed, 89 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0f772b35/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index cc599a4..86eea55 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -20,15 +20,18 @@ package org.apache.kylin.metadata.model;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.model.JoinsTree.Chain;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -294,6 +297,11 @@ public class DataModelDesc extends RootPersistentEntity {
 initJoinsTree();
 initDimensionsAndMetrics();
 initPartitionDesc();
+
+boolean reinit = validate();
+if (reinit) { // model slightly changed by validate() and must init() 
again
+init(config, tables);
+}
 }
 
 private void initJoinTablesForUpgrade() {
@@ -443,6 +451,74 @@ public class DataModelDesc extends RootPersistentEntity {
 joinsTree = new JoinsTree(rootFactTableRef, joins);
 }
 
+private boolean validate() {
+Set mcols = new HashSet<>();
+for (String m : metrics) {
+mcols.add(findColumn(m));
+}
+
+// validate no dup between dimensions/metrics
+for (ModelDimensionDesc dim : dimensions) {
+String table = dim.getTable();
+for (String c : dim.getColumns()) {
+TblColRef dcol = findColumn(table, c);
+if (mcols.contains(dcol))
+throw new IllegalStateException(dcol + " cannot be both 
dimension and metrics at the same time in " + this);
+}
+}
+
+// validate PK/FK are in dimensions
+boolean pkfkDimAmended = false;
+for (Chain chain : joinsTree.tableChains.values()) {
+pkfkDimAmended = validatePkFkDim(chain.join, mcols) || 
pkfkDimAmended;
+}
+return pkfkDimAmended;
+}
+
+private boolean validatePkFkDim(JoinDesc join, Set mcols) {
+if (join == null)
+return false;
+
+boolean pkfkDimAmended = false;
+
+for (TblColRef c : join.getForeignKeyColumns()) {
+if (!mcols.contains(c)) {
+pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended;
+}
+}
+for (TblColRef c : join.getPrimaryKeyColumns()) {
+if (!mcols.contains(c)) {
+pkfkDimAmended = validatePkFkDim(c) || pkfkDimAmended;
+}
+}
+return pkfkDimAmended;
+}
+
+private boolean validatePkFkDim(TblColRef c) {
+String t = c.getTableAlias();
+ModelDimensionDesc dimDesc = null;
+for (ModelDimensionDesc dim : dimensions) {
+if (dim.getTable().equals(t)) {
+dimDesc = dim;
+break;
+}
+}
+
+if (dimDesc == null) {
+dimDesc = new ModelDimensionDesc();
+dimDesc.setTable(t);
+dimDesc.setColumns(new String[0]);
+}
+
+if (ArrayUtils.contains(dimDesc.getColumns(), c.getName()) == 

kylin git commit: KYLIN-2421 fix unit test

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/spark-it b0a3a28eb -> b55250e24


KYLIN-2421 fix unit test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b55250e2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b55250e2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b55250e2

Branch: refs/heads/spark-it
Commit: b55250e24d9f4b9ce276e8fc67cccde99f4f52e3
Parents: b0a3a28
Author: shaofengshi 
Authored: Sat Feb 4 14:43:36 2017 +0800
Committer: shaofengshi 
Committed: Sat Feb 4 14:43:36 2017 +0800

--
 core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b55250e2/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
--
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java 
b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 86ea1df..20ee43e 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -48,6 +48,7 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -102,6 +103,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase 
{
 this.cleanupTestMetadata();
 }
 
+@Ignore ("To enable spark in IT, the inner cube removed the percentile 
measure, so ignore this test")
 @Test
 public void testCiCube() {
 CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig());

[15/39] kylin git commit: KYLIN 2396 GUI for Percentile pre aggregation implementation

[shaofengshi]

KYLIN 2396 GUI for Percentile pre aggregation implementation

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28673a81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28673a81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28673a81

Branch: refs/heads/spark-it
Commit: 28673a8126ec2ecff8c125c0ecf887ce2565362c
Parents: 6aaf667
Author: chenzhx <346839...@qq.com>
Authored: Fri Jan 20 18:00:20 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 11:32:46 2017 +0800

--
 webapp/app/js/controllers/cubeMeasures.js  | 18 +++
 webapp/app/js/model/cubeConfig.js  |  2 +-
 webapp/app/partials/cubeDesigner/measures.html | 54 -
 3 files changed, 40 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index c8548ce..7c155fe 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -80,6 +80,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   $scope.updateMeasureStatus.editIndex = index;
 }
 $scope.nextParameters = [];
+$scope.measureParamValueColumn=$scope.getCommonMetricColumns();
 $scope.newMeasure = (!!measure)? jQuery.extend(true, 
{},measure):CubeDescModel.createMeasure();
 if(!!measure && measure.function.parameter.next_parameter){
   $scope.nextPara.value = measure.function.parameter.next_parameter.value;
@@ -93,6 +94,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 }else{
   $scope.newMeasure.showDim=false;
 }
+$scope.measureParamValueUpdate();
 if($scope.newMeasure.function.expression=="TOP_N"){
   $scope.convertedColumns=[];
   if($scope.newMeasure.function.configuration==null){
@@ -326,6 +328,19 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   return false;
 }
   }
+
+  $scope.measureParamValueUpdate = function(){
+if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && 
$scope.newMeasure.showDim==true){
+   $scope.measureParamValueColumn=$scope.getAllModelDimMeasureColumns();
+}
+if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && 
$scope.newMeasure.showDim==false){
+   $scope.measureParamValueColumn=$scope.getCommonMetricColumns();
+}
+if($scope.newMeasure.function.expression == 'EXTENDED_COLUMN'){
+  $scope.measureParamValueColumn=$scope.getExtendedHostColumn();
+}
+  }
+
   //map right return type for param
   $scope.measureReturnTypeUpdate = function(){
 
@@ -393,6 +408,9 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 case "COUNT":
   $scope.newMeasure.function.returntype = "bigint";
   break;
+case "PERCENTILE":
+  $scope.newMeasure.function.returntype = "percentile(100)";
+  break;
 default:
   $scope.newMeasure.function.returntype = "";
   break;

http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/model/cubeConfig.js
--
diff --git a/webapp/app/js/model/cubeConfig.js 
b/webapp/app/js/model/cubeConfig.js
index 87413dc..35af1fc 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -20,7 +20,7 @@ KylinApp.constant('cubeConfig', {
 
   //~ Define metadata & class
   measureParamType: ['column', 'constant'],
-  measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 
'RAW','EXTENDED_COLUMN'],
+  measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 
'RAW','EXTENDED_COLUMN','PERCENTILE'],
   dimensionDataTypes: ["string", "tinyint", "int", "bigint", "date"],
   cubePartitionTypes: ['APPEND'],
   joinTypes: [

http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index 4d2d8ab..4a48843 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -104,7 +104,7 @@
   
   
   
   
@@ -144,35 +144,23 @@
 
 
   
-

[19/39] kylin git commit: KYLIN-2413 a new profile for cdh binary packages

[shaofengshi]

KYLIN-2413 a new profile for cdh binary packages


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb73da7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb73da7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb73da7a

Branch: refs/heads/spark-it
Commit: eb73da7a0b27540af54fec23b561b390405620c5
Parents: 169c3de
Author: lidongsjtu 
Authored: Mon Jan 23 11:30:13 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 13:03:29 2017 +0800

--
 build/script/build.sh   |   2 +-
 build/script/package.sh |  11 +-
 .../kylin/engine/mr/steps/MockupMapContext.java |  11 +-
 pom.xml | 114 +++
 .../storage/hbase/steps/MockupMapContext.java   |  17 +--
 tool/pom.xml|  10 ++
 6 files changed, 145 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/build/script/build.sh
--
diff --git a/build/script/build.sh b/build/script/build.sh
index c68ca94..458c612 100755
--- a/build/script/build.sh
+++ b/build/script/build.sh
@@ -22,7 +22,7 @@ echo "package back-end"
 dir=$(dirname ${0})
 cd ${dir}/../..
 
-mvn clean install -DskipTests   || { exit 1; }
+mvn clean install -DskipTests $@ || { exit 1; }
 
 #package webapp
 echo 'package front-end'

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/build/script/package.sh
--
diff --git a/build/script/package.sh b/build/script/package.sh
index c850ec3..a58a5ce 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -74,9 +74,8 @@ cat << EOF > build/commit_SHA1
 EOF
 git rev-parse HEAD >> build/commit_SHA1
 
-sh build/script/build.sh || { exit 1; }
-sh build/script/download-tomcat.sh || { exit 1; }
-sh build/script/download-spark.sh || { exit 1; }
-sh build/script/prepare.sh || { exit 1; }
-sh build/script/compress.sh || { exit 1; }
-
+sh build/script/build.sh $@ || { exit 1; }
+sh build/script/download-tomcat.sh  || { exit 1; }
+sh build/script/download-spark.sh   || { exit 1; }
+sh build/script/prepare.sh  || { exit 1; }
+sh build/script/compress.sh || { exit 1; }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index dfbba14..92115c9 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -47,7 +47,7 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
  * 
  */
 @SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext implements MapContext{
+public class MockupMapContext implements MapContext {
 
 private Configuration hconf;
 
@@ -60,7 +60,7 @@ public class MockupMapContext implements MapContext{
 return new WrappedMapper().getMapContext(new MockupMapContext(hconf, 
outKV));
 }
 
-public MockupMapContext(Configuration hconf, Object[] outKV){
+public MockupMapContext(Configuration hconf, Object[] outKV) {
 this.hconf = hconf;
 this.outKV = outKV;
 }
@@ -109,7 +109,6 @@ public class MockupMapContext implements MapContext{
 throw new NotImplementedException();
 }
 
-@Override
 public float getProgress() {
 throw new NotImplementedException();
 }
@@ -224,7 +223,6 @@ public class MockupMapContext implements MapContext{
 throw new NotImplementedException();
 }
 
-@Override
 public boolean getTaskCleanupNeeded() {
 throw new NotImplementedException();
 }
@@ -239,7 +237,6 @@ public class MockupMapContext implements MapContext{
 throw new NotImplementedException();
 }
 
-@Override
 public IntegerRanges getProfileTaskRange(boolean isMap) {
 throw new NotImplementedException();
 }
@@ -318,4 +315,8 @@ public class MockupMapContext implements MapContext{
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+public boolean userClassesTakesPrecedence() {
+throw new NotImplementedException();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 

[31/39] kylin git commit: KYLIN-2374 fix path bug in HDFSResourceStore

[shaofengshi]

KYLIN-2374 fix path bug in HDFSResourceStore

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56293411
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56293411
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56293411

Branch: refs/heads/spark-it
Commit: 5629341122abe18d9576f71a0ea99ac83c327c21
Parents: a86569a
Author: xiefan46 <958034...@qq.com>
Authored: Wed Jan 25 10:41:40 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Jan 25 11:19:13 2017 +0800

--
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 23 +++-
 1 file changed, 13 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56293411/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index 0ad6964..38acfb0 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -18,7 +18,15 @@
 
 package org.apache.kylin.storage.hdfs;
 
-import com.google.common.collect.Lists;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -32,14 +40,7 @@ import org.apache.kylin.common.util.HadoopUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Lists;
 
 public class HDFSResourceStore extends ResourceStore {
 
@@ -63,7 +64,7 @@ public class HDFSResourceStore extends ResourceStore {
 int cut = metadataUrl.indexOf('@');
 String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : 
metadataUrl.substring(0, cut);
 String hdfsUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 
1);
-if(!hdfsUrl.equals("hdfs"))
+if (!hdfsUrl.equals("hdfs"))
 throw new IOException("Can not create HDFSResourceStore. Url not 
match. Url:" + hdfsUrl);
 metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME;
 logger.info("meta dir name :" + metaDirName);
@@ -245,6 +246,8 @@ public class HDFSResourceStore extends ResourceStore {
 }
 
 private Path getRealHDFSPath(String resourcePath) {
+if (resourcePath.equals("/"))
+return this.hdfsMetaPath;
 if (resourcePath.startsWith("/") && resourcePath.length() > 1)
 resourcePath = resourcePath.substring(1, resourcePath.length());
 return new Path(this.hdfsMetaPath, resourcePath);

[18/39] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[shaofengshi]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/169c3de6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/169c3de6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/169c3de6

Branch: refs/heads/spark-it
Commit: 169c3de6456f394d049e9550d65a98f6aa0a316f
Parents: 68e87f0
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 13:01:20 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index e02bf19..cbec4f6 100644
--- 

[32/39] kylin git commit: KYLIN-2374 minor, rename zookeeper config properties

[shaofengshi]

KYLIN-2374 minor,rename zookeeper config properties

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2a59c43
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2a59c43
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2a59c43

Branch: refs/heads/spark-it
Commit: a2a59c4303a29de308beb7b5c8300f78023313dd
Parents: 5629341
Author: xiefan46 <958034...@qq.com>
Authored: Tue Jan 24 18:57:07 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Jan 25 14:21:37 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java  | 4 ++--
 examples/test_case_data/sandbox/kylin.properties| 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a2a59c43/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 07cb124..5932197 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -930,11 +930,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 //zoo keeper
 public String getZooKeeperHost() {
-return getOptional("kylin.storage-zookeeper.host", "localhost");
+return getOptional("kylin.storage.lock-manager-zk-host", "localhost");
 }
 
 public String getZooKeeperPort() {
-return getOptional("kylin.storage-zookeeper.port", "2181");
+return getOptional("kylin.storage.lock-manager-zk-port", "2181");
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a2a59c43/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index fa0c6bc..6cb5148 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -55,6 +55,9 @@ kylin.env.hdfs-working-dir=/kylin
 
 kylin.engine.mr.reduce-input-mb=500
 
+#zoo keeper host in lock manager
+kylin.storage.lock-manager-zk-host=sandbox
+
 ### JOB ###
 
 # max job retry on error, default 0: no retry
@@ -179,5 +182,3 @@ 
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
 
 
-#zoo keeper
-kylin.storage-zookeeper.host=sandbox
\ No newline at end of file

[10/39] kylin git commit: Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs"

[shaofengshi]

Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs"

This reverts commit 7de8aa1203a72bad105ed692f7100535939b03af.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7576a09f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7576a09f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7576a09f

Branch: refs/heads/spark-it
Commit: 7576a09f29b75f2eb80a1b5ca9c65fe5c81b1144
Parents: 7de8aa1
Author: shaofengshi 
Authored: Sun Jan 22 11:38:17 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 11:38:17 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 --
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 +--
 .../mr/steps/FactDistinctColumnsReducer.java| 240 ---
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 187 insertions(+), 175 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,10 +26,8 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -142,18 +140,4 @@ public class HadoopUtil {
 }
 }
 
-public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
-FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
-@Override
-public boolean accept(Path path) {
-return path.getName().startsWith(filter);
-}
-});
-
-if (fileStatus.length == 1) {
-return fileStatus[0].getPath();
-} else {
-return null;
-}
-}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index c34a904..696b22a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
+return getRealizationRootPath(jobId) + "/statistics";
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 602b4bb..0281539 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,16 +53,9 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
+String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
-String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
-
-String CFG_OUTPUT_COLUMN = 

[37/39] kylin git commit: KYLIN-2418 drop unused useSandbox

[shaofengshi]

KYLIN-2418 drop unused useSandbox


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/546f88f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/546f88f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/546f88f3

Branch: refs/heads/spark-it
Commit: 546f88f38ca017773c17ccaf2cf4812b0c7c4666
Parents: 0582512
Author: Yang Li 
Authored: Mon Jan 30 09:07:29 2017 +0800
Committer: Yang Li 
Committed: Tue Jan 31 08:11:41 2017 +0800

--
 .../kylin/common/persistence/FileResourceStore.java   |  4 
 .../org/apache/kylin/common/persistence/ResourceTool.java | 10 ++
 kylin-it/pom.xml  |  6 --
 pom.xml   |  8 
 .../apache/kylin/storage/hbase/HBaseResourceStore.java|  5 +
 5 files changed, 15 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index dcd3f38..3e012f5 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -160,4 +160,8 @@ public class FileResourceStore extends ResourceStore {
 return new File(root, resPath);
 }
 
+@Override
+public String toString() {
+return root.getAbsolutePath();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index b3aac09..6ba68ae 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -133,22 +133,24 @@ public class ResourceTool {
 ResourceStore src = ResourceStore.getStore(srcConfig);
 ResourceStore dst = ResourceStore.getStore(dstConfig);
 
+logger.info("Copy from {} to {}", src, dst);
+
 copyR(src, dst, path);
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, 
List paths) throws IOException {
 ResourceStore src = ResourceStore.getStore(srcConfig);
 ResourceStore dst = ResourceStore.getStore(dstConfig);
+
+logger.info("Copy from {} to {}", src, dst);
+
 for (String path : paths) {
 copyR(src, dst, path);
 }
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) 
throws IOException {
-
-ResourceStore src = ResourceStore.getStore(srcConfig);
-ResourceStore dst = ResourceStore.getStore(dstConfig);
-copyR(src, dst, "/");
+copy(srcConfig, dstConfig, "/");
 }
 
 public static void copyR(ResourceStore src, ResourceStore dst, String 
path) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index f88db9f..9662806 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -272,10 +272,6 @@
 
 
 
-useSandbox
-true
-
-
 log4j.configuration
 
file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
 
@@ -298,7 +294,6 @@
 test
 java
 
--DuseSandbox=true
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
 
-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
@@ -320,7 +315,6 @@
 test
 java
  

[16/39] kylin git commit: add back calcite linq4j

[shaofengshi]

add back calcite linq4j


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68e87f08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68e87f08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68e87f08

Branch: refs/heads/spark-it
Commit: 68e87f08255caf33dd7946f43b3393372a73c68b
Parents: b56bc62
Author: Hongbin Ma 
Authored: Mon Jan 23 11:54:54 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 11:54:54 2017 +0800

--
 pom.xml | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/68e87f08/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 57b7752..e590fd7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -450,6 +450,11 @@
 
 
 
+org.apache.calcite
+calcite-linq4j
+${calcite.version}
+
+
 org.apache.calcite.avatica
 avatica
 ${avatica.version}

[06/39] kylin git commit: KYLIN 2370 Refine unload and reload table

[shaofengshi]

KYLIN 2370 Refine unload and reload table

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a853a7c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a853a7c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a853a7c4

Branch: refs/heads/spark-it
Commit: a853a7c4b6b7d42cc3ffd066b618f5ea72ca0d9f
Parents: 85a1eb3
Author: chenzhx <346839...@qq.com>
Authored: Mon Jan 9 15:09:16 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 16:21:11 2017 +0800

--
 webapp/app/css/AdminLTE.css |  27 +++
 webapp/app/js/controllers/sourceMeta.js | 175 ---
 .../js/directives/kylin_abn_tree_directive.js   |   7 +-
 .../app/partials/tables/source_table_tree.html  |   6 +-
 webapp/app/partials/tables/table_unload.html|  33 
 5 files changed, 111 insertions(+), 137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/css/AdminLTE.css
--
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index 857dbf7..c7740d1 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -4800,3 +4800,30 @@ Gradient Background colors
 white-space: normal !important;
   }
 }
+
+.abn-tree .abn-tree-row .tree-table-btn{
+width: 4%;
+float: right;
+border-radius: 5px;
+margin-right: 5px;
+padding: 3px 16px 2px 5px;
+color: #ff;
+position:static;
+}
+.abn-tree .abn-tree-row .tree-table-btn .tooltip{
+overflow:visible;
+}
+.abn-tree .abn-tree-row .btn-info:hover{
+   background-color:#269abc;
+}
+.abn-tree > .abn-tree-row.active > .btn-info{
+background-color:#269abc;
+border-left-color:#269abc;
+}
+.abn-tree .abn-tree-row .btn-success:hover{
+background-color:#008d4c;
+}
+.abn-tree > .abn-tree-row.active > .btn-success{
+background-color:#008d4c;
+border-left-color:#008d4c;
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/js/controllers/sourceMeta.js
--
diff --git a/webapp/app/js/controllers/sourceMeta.js 
b/webapp/app/js/controllers/sourceMeta.js
index a53a35f..5421673 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -132,31 +132,83 @@ KylinApp
   });
 };
 
-$scope.openUnLoadModal = function () {
-  if(!$scope.projectModel.selectedProject){
-SweetAlert.swal('Oops...', "Please select a project.", 'info');
+$scope.reloadTable = function (tableName,projectName){
+  loadingRequest.show();
+  TableService.loadHiveTable({tableName: tableName, action: projectName}, 
{calculate: $scope.isCalculate}, function (result) {
+var loadTableInfo = "";
+angular.forEach(result['result.loaded'], function (table) {
+  loadTableInfo += "\n" + table;
+})
+var unloadedTableInfo = "";
+angular.forEach(result['result.unloaded'], function (table) {
+  unloadedTableInfo += "\n" + table;
+})
+if (result['result.unloaded'].length != 0 && 
result['result.loaded'].length == 0) {
+  SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + 
unloadedTableInfo, 'error');
+}
+if (result['result.loaded'].length != 0 && 
result['result.unloaded'].length == 0) {
+  SweetAlert.swal('Success!', 'The following table(s) have been 
successfully loaded: ' + loadTableInfo, 'success');
+}
+if (result['result.loaded'].length != 0 && 
result['result.unloaded'].length != 0) {
+  SweetAlert.swal('Partial loaded!', 'The following table(s) have been 
successfully loaded: ' + loadTableInfo + "\n\n Failed to load following 
table(s):" + unloadedTableInfo, 'warning');
+}
+loadingRequest.hide();
+$scope.aceSrcTbLoaded(true);
+  }, function (e) {
+if (e.data && e.data.exception) {
+  var message = e.data.exception;
+  var msg = !!(message) ? message : 'Failed to take action.';
+  SweetAlert.swal('Oops...', msg, 'error');
+} else {
+  SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+}
+loadingRequest.hide();
+  })
+}
+
+
+
+$scope.removeList = function (tableName,projectName) {
+  if (tableName.trim() === "") {
+SweetAlert.swal('', 'Please input table(s) you want to unload.', 
'info');
 return;
   }
-  $modal.open({
-templateUrl: 'removeHiveTable.html',
-controller: ModalInstanceCtrl,
-backdrop : 'static',
-resolve: {
-  tableNames: function () {
-return $scope.tableNames;
- 

[27/39] kylin git commit: Add HDFSResourceStore to ResourceStore

[shaofengshi]

Add HDFSResourceStore to ResourceStore

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c33e38f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c33e38f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c33e38f

Branch: refs/heads/spark-it
Commit: 4c33e38f3e6c33438ef3bb830875b8ed74ffac5b
Parents: d3276e2
Author: xiefan46 <958034...@qq.com>
Authored: Mon Jan 23 10:51:12 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../kylin/common/persistence/ResourceStore.java   |  5 +
 .../kylin/storage/hdfs/ITHDFSResourceStoreTest.java   | 13 -
 .../apache/kylin/storage/hdfs/ITLockManagerTest.java  | 14 +-
 .../kylin/storage/hbase/HBaseResourceStore.java   |  3 ++-
 .../apache/kylin/storage/hdfs/HDFSResourceStore.java  |  4 +++-
 5 files changed, 23 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 9549569..25a0801 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -75,6 +75,11 @@ abstract public class ResourceStore {
 } catch (Throwable e) {
 logger.warn("Failed to load HBaseResourceStore impl class: " + 
e.toString());
 }
+try {
+
knownImpl.add(ClassUtil.forName("org.apache.kylin.storage.hdfs.HDFSResourceStore",
 ResourceStore.class));
+} catch (Throwable e) {
+logger.warn("Failed to load HDFSResourceStore impl class: " + 
e.toString());
+}
 }
 return knownImpl;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 27d8a3c..47afb01 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -19,17 +19,12 @@
 package org.apache.kylin.storage.hdfs;
 
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.ResourceStoreTest;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-
-/**
- * Created by xiefan on 17-1-10.
- */
 public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
 
 KylinConfig kylinConfig;
@@ -45,13 +40,13 @@ public class ITHDFSResourceStoreTest extends 
HBaseMetadataTestCase {
 this.cleanupTestMetadata();
 }
 
-
 @Test
 public void testResourceStoreBasic() throws Exception {
-KylinConfig config = KylinConfig.getInstanceFromEnv();
-ResourceStore store = new HDFSResourceStore(config);
+String oldUrl = kylinConfig.getMetadataUrl();
+kylinConfig.setProperty("kylin.metadata.url", 
"kylin_default_instance@hdfs");
+HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
 ResourceStoreTest.testAStore(store);
+kylinConfig.setProperty("kylin.metadata.url", oldUrl);
 }
 
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
index 2b58d30..a6cf43b 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
@@ -27,6 +27,8 @@ import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -58,6 +60,8 @@ public class ITLockManagerTest extends 

[07/39] kylin git commit: KYLIN-2308 Allow user to set more columnFamily in web

[shaofengshi]

KYLIN-2308 Allow user to set more columnFamily in web

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8331d8d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8331d8d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8331d8d0

Branch: refs/heads/spark-it
Commit: 8331d8d0ad37d24bdbfa90d1032bcdd063aa0a59
Parents: a853a7c
Author: kangkaisen 
Authored: Sat Jan 7 15:34:57 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 17:25:28 2017 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 75 +++-
 webapp/app/js/controllers/cubeEdit.js   | 51 ---
 webapp/app/js/controllers/cubeSchema.js | 13 +++
 webapp/app/js/filters/filter.js | 22 +++--
 .../cubeDesigner/advanced_settings.html | 92 +++-
 5 files changed, 194 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 760133a..39d36b0 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -278,11 +278,84 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 $scope.isReuse=!$scope.isReuse;
   }
 
-  $scope.removeDictionaries =  function(arr,element){
+  $scope.removeElement =  function(arr,element){
 var index = arr.indexOf(element);
 if (index > -1) {
   arr.splice(index, 1);
 }
   };
 
+  $scope.newColFamily = function (index) {
+return {
+"name": "F" + index,
+"columns": [
+  {
+"qualifier": "M",
+"measure_refs": []
+  }
+]
+  };
+  };
+
+  $scope.initColumnFamily = function () {
+$scope.cubeMetaFrame.hbase_mapping.column_family = [];
+var normalMeasures = [], distinctCountMeasures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  if (measure.function.expression === 'COUNT_DISTINCT') {
+distinctCountMeasures.push(measure);
+  } else {
+normalMeasures.push(measure);
+  }
+});
+if (normalMeasures.length > 0) {
+  var nmcf = $scope.newColFamily(1);
+  angular.forEach(normalMeasures, function (normalM, index) {
+nmcf.columns[0].measure_refs.push(normalM.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(nmcf);
+}
+
+if (distinctCountMeasures.length > 0) {
+  var dccf = $scope.newColFamily(2);
+  angular.forEach(distinctCountMeasures, function (dcm, index) {
+dccf.columns[0].measure_refs.push(dcm.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(dccf);
+}
+  };
+
+  $scope.getAllMeasureNames = function () {
+var measures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  measures.push(measure.name);
+});
+return measures;
+  };
+
+  $scope.getAssignedMeasureNames = function () {
+var assignedMeasures = [];
+angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function 
(colFamily, index) {
+  angular.forEach(colFamily.columns[0].measure_refs, function (measure, 
index) {
+assignedMeasures.push(measure);
+  });
+});
+return assignedMeasures;
+  };
+
+  if ($scope.getAllMeasureNames().length != 
$scope.getAssignedMeasureNames().length) {
+$scope.initColumnFamily();
+  }
+
+
+  $scope.addColumnFamily = function () {
+var colFamily = 
$scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 
1);
+$scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
+  };
+
+  $scope.refreshColumnFamily = function (column_familys, index, colFamily) {
+if (column_familys) {
+  column_familys[index] = colFamily;
+}
+  };
+
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index edbb421..da19b22 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -284,25 +284,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return type;
   };
 
-  var ColFamily = function () {
-var index = 1;
-return function () {
-  var newColFamily =
-  {
-"name": "f" + index,
-"columns": [
-  {
-

[03/39] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[shaofengshi]

KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dfb5fac9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dfb5fac9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dfb5fac9

Branch: refs/heads/spark-it
Commit: dfb5fac9f0d9c13f1128206b73e683b515fe3d72
Parents: 61833d9
Author: Billy Liu 
Authored: Thu Jan 19 14:54:20 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 19 14:54:20 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3941 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2430 insertions(+), 1571 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b916df2..1b327fe 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -36,7 +36,16 @@
 
 org.apache.calcite
 calcite-core
+
+
+org.apache.calcite.avatica
+avatica-core
+
+
+
+
+org.apache.calcite.avatica
+avatica
 
 
-
 

[23/39] kylin git commit: KYLIN-2217 use column’s identity as path name and add more logging info

[shaofengshi]

KYLIN-2217 use column’s identity as path name and add more logging info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e217ae84
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e217ae84
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e217ae84

Branch: refs/heads/spark-it
Commit: e217ae84e1bd163658176093de208fcdf3bdf43c
Parents: f6dda4f
Author: shaofengshi 
Authored: Sun Jan 22 09:43:10 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../java/org/apache/kylin/common/util/HadoopUtil.java|  4 
 .../apache/kylin/cube/cli/DictionaryGeneratorCLI.java|  3 +++
 .../org/apache/kylin/engine/mr/SortedColumnDFSFile.java  |  4 
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java |  2 +-
 .../kylin/engine/mr/steps/CreateDictionaryJob.java   |  9 +++--
 .../engine/mr/steps/FactDistinctColumnsReducer.java  | 11 +++
 .../engine/mr/steps/UpdateCubeInfoAfterBuildStep.java|  4 ++--
 .../hive/cardinality/HiveColumnCardinalityJob.java   |  3 +++
 8 files changed, 31 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..f242515 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -143,6 +143,10 @@ public class HadoopUtil {
 }
 
 public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+if (fs.exists(baseDir) == false) {
+return null;
+}
+
 FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
 @Override
 public boolean accept(Path path) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 3e1ab0d..2e5a38e 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -63,11 +63,14 @@ public class DictionaryGeneratorCLI {
 if (dictProvider != null) {
 Dictionary dict = dictProvider.getDictionary(col);
 if (dict != null) {
+logger.debug("Dict for '" + col.getName() + "' has already 
been built, save it");
 cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);
 } else {
+logger.debug("Dict for '" + col.getName() + "' not 
pre-built, build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 } else {
+logger.debug("Dict for '" + col.getName() + "' not pre-built, 
build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
index d3f5cdc..f396b5a 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
@@ -128,4 +128,8 @@ public class SortedColumnDFSFile implements ReadableTable {
 return comparator;
 }
 
+@Override
+public String toString() {
+return dfsPath;
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 567c1d0..44686d6 100644
--- 

[14/39] kylin git commit: KYLIN 2396 GUI for Percentile pre aggregation implementation

[shaofengshi]

KYLIN 2396 GUI for Percentile pre aggregation implementation

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b56bc622
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b56bc622
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b56bc622

Branch: refs/heads/spark-it
Commit: b56bc622086495f7d0047d01af57efc919b2ecce
Parents: 28673a8
Author: luguosheng <550175...@qq.com>
Authored: Mon Jan 23 11:24:39 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 11:32:46 2017 +0800

--
 webapp/app/js/controllers/cubeMeasures.js  | 2 ++
 webapp/app/partials/cubeDesigner/measures.html | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 7c155fe..91c38f2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -355,6 +355,8 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   $scope.newMeasure.function.parameter.type= 'column';
   $scope.newMeasure.function.returntype = "extendedcolumn(100)";
   return;
+}else if($scope.newMeasure.function.expression=='PERCENTILE'){
+  $scope.newMeasure.function.parameter.type= 'column';
 }else{
   $scope.nextParameterInit();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index 4a48843..b7475b6 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -116,7 +116,7 @@
   
   Param Type
   
-  
   
   constant
+ ng-if="newMeasure.function.expression 
== 'COUNT'||newMeasure.function.expression == 'PERCENTILE'"
+ 
>{{newMeasure.function.parameter.type}}
   
   
   

[08/39] kylin git commit: KYLIN-2396 remove unused code

[shaofengshi]

KYLIN-2396 remove unused code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2229c9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2229c9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2229c9c

Branch: refs/heads/spark-it
Commit: c2229c9c7f83fe5a28e564c37ada5794a67ee3d8
Parents: 8331d8d
Author: lidongsjtu 
Authored: Sat Jan 21 19:52:42 2017 +0800
Committer: lidongsjtu 
Committed: Sat Jan 21 19:52:42 2017 +0800

--
 .../measure/percentile/PercentileContUdf.java   | 37 
 .../kylin/rest/controller/BasicController.java  | 22 
 .../rest/controller/DiagnosisController.java| 19 --
 3 files changed, 22 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2229c9c/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
deleted file mode 100644
index 4ef6b75..000
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.percentile;
-
-public class PercentileContUdf {
-public static double init() {
-return 0;
-}
-
-public static double add(double accumulator, double v, double r) {
-return 0;
-}
-
-public static double merge(double accumulator0, double accumulator1) {
-return 0;
-}
-
-public static double result(long accumulator) {
-return 0;
-}
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2229c9c/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
index f61492d..6b4c9fd 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
@@ -18,11 +18,20 @@
 
 package org.apache.kylin.rest.controller;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.exception.NotFoundException;
 import org.apache.kylin.rest.response.ErrorResponse;
 import org.slf4j.Logger;
@@ -74,4 +83,17 @@ public class BasicController {
 }
 }
 
+protected void setDownloadResponse(String downloadFile, final 
HttpServletResponse response) {
+File file = new File(downloadFile);
+try (InputStream fileInputStream = new FileInputStream(file); 
OutputStream output = response.getOutputStream();) {
+response.reset();
+response.setContentType("application/octet-stream");
+response.setContentLength((int) (file.length()));
+response.setHeader("Content-Disposition", "attachment; 
filename=\"" + file.getName() + "\"");
+IOUtils.copyLarge(fileInputStream, output);
+output.flush();
+} catch (IOException e) {
+throw new InternalErrorException("Failed to download file: " + 
e.getMessage(), e);
+}
+}
 }

[28/39] kylin git commit: KYLIN-2415 Change back default metadata name to kylin_metadata

[shaofengshi]

KYLIN-2415 Change back default metadata name to kylin_metadata


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6d247c2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6d247c2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6d247c2

Branch: refs/heads/spark-it
Commit: b6d247c2c1412d7eb1024048ae7aad0ba5d1ea82
Parents: 971bf80
Author: shaofengshi 
Authored: Tue Jan 24 13:43:04 2017 +0800
Committer: shaofengshi 
Committed: Tue Jan 24 13:43:04 2017 +0800

--
 build/conf/kylin.properties| 2 +-
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
 .../apache/kylin/metadata/realization/IRealizationConstants.java   | 2 +-
 examples/test_case_data/sandbox/kylin.properties   | 2 +-
 .../org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java | 2 +-
 .../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java| 2 +-
 .../org/apache/kylin/storage/hbase/util/StorageCleanupJob.java | 2 +-
 .../main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java | 2 +-
 tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java| 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 196a711..eceb886 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -18,7 +18,7 @@
 ### METADATA | ENV ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_default_instance@hbase
+kylin.metadata.url=kylin_metadata@hbase
 
 # Working folder in HDFS, make sure user has the right access to the hdfs 
directory
 kylin.env.hdfs-working-dir=/kylin

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 75b38ff..07cb124 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -216,7 +216,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 public String getMetadataUrlPrefix() {
 String metadataUrl = getMetadataUrl();
-String defaultPrefix = "kylin_default_instance";
+String defaultPrefix = "kylin_metadata";
 
 if (metadataUrl.endsWith("@hbase")) {
 int cut = metadataUrl.lastIndexOf('@');

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index adc53a1..2a3b01e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -28,7 +28,7 @@ public class IRealizationConstants {
 
 /**
  * For each cube htable, we leverage htable's metadata to keep track of
- * which kylin server(represented by its kylin_default_instance prefix) 
owns this htable
+ * which kylin server(represented by its kylin_metadata prefix) owns this 
htable
  */
 public final static String HTableTag = "KYLIN_HOST";
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 2c2da91..fa0c6bc 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -39,7 +39,7 @@ kylin.source.hive.client=cli
 ### STORAGE ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_default_instance@hbase
+kylin.metadata.url=kylin_metadata@hbase
 
 
 # The storage for final cube file in hbase

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 

[36/39] kylin git commit: minor, make DefaultSchedulerTest.testDiscard() more stable

[shaofengshi]

minor, make DefaultSchedulerTest.testDiscard() more stable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0582512e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0582512e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0582512e

Branch: refs/heads/spark-it
Commit: 0582512ea06bcd1c662903ff69012ba1cc7cc33c
Parents: c3fff6d
Author: Li Yang 
Authored: Thu Jan 26 11:49:04 2017 +0800
Committer: Li Yang 
Committed: Thu Jan 26 11:49:56 2017 +0800

--
 .../apache/kylin/job/SelfStopExecutable.java| 31 
 .../impl/threadpool/DefaultSchedulerTest.java   |  5 ++--
 2 files changed, 27 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java 
b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
index b4f6a98..9a3eb48 100644
--- a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
+++ b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
@@ -26,21 +26,40 @@ import org.apache.kylin.job.execution.ExecuteResult;
  */
 public class SelfStopExecutable extends BaseTestExecutable {
 
+volatile boolean doingWork;
+
 public SelfStopExecutable() {
 super();
 }
 
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
+doingWork = true;
 try {
-Thread.sleep(5000);
-} catch (InterruptedException e) {
-}
-if (isDiscarded()) {
-return new ExecuteResult(ExecuteResult.State.STOPPED, "stopped");
-} else {
+for (int i = 0; i < 20; i++) {
+sleepOneSecond();
+
+if (isDiscarded())
+return new ExecuteResult(ExecuteResult.State.STOPPED, 
"stopped");
+}
+
 return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+} finally {
+doingWork = false;
+}
+}
+
+private void sleepOneSecond() {
+try {
+Thread.sleep(1000);
+} catch (InterruptedException e) {
+throw new RuntimeException(e);
 }
 }
 
+public void waitForDoWork() {
+while (doingWork) {
+sleepOneSecond();
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index bcd6a59..2416311 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -97,7 +97,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest {
 @Test
 public void testDiscard() throws Exception {
 DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SelfStopExecutable();
+SelfStopExecutable task1 = new SelfStopExecutable();
 job.addTask(task1);
 jobService.addJob(job);
 waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
@@ -105,8 +105,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest 
{
 waitForJobFinish(job.getId());
 Assert.assertEquals(ExecutableState.DISCARDED, 
jobService.getOutput(job.getId()).getState());
 Assert.assertEquals(ExecutableState.DISCARDED, 
jobService.getOutput(task1.getId()).getState());
-Thread.sleep(5000);
-System.out.println(job);
+task1.waitForDoWork();
 }
 
 @SuppressWarnings("rawtypes")

[33/39] kylin git commit: KYLIN-2374 code review

[shaofengshi]

KYLIN-2374 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eae37ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eae37ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eae37ef

Branch: refs/heads/spark-it
Commit: 5eae37ef18ca51027c6bb2cfd3410fefc7982f2a
Parents: a2a59c4
Author: shaofengshi 
Authored: Thu Jan 26 09:55:48 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 09:55:48 2017 +0800

--
 build/conf/kylin.properties |  3 +-
 build/deploy/spark-defaults.conf|  1 -
 .../apache/kylin/common/KylinConfigBase.java|  8 --
 .../kylin/common/persistence/ResourceStore.java |  3 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |  2 +-
 .../ExtendedColumnMeasureType.java  |  8 +-
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 36 +++-
 .../kylin/storage/hbase/HBaseResourceStore.java |  3 +-
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 90 +++-
 9 files changed, 97 insertions(+), 57 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index eceb886..43ea17d 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -211,8 +211,9 @@ kylin.engine.spark-conf.spark.executor.memory=4G
 kylin.engine.spark-conf.spark.executor.cores=4
 kylin.engine.spark-conf.spark.executor.instances=8
 kylin.engine.spark-conf.spark.storage.memoryFraction=0.3
-kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.eventLog.enabled=true
 kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
 ## manually upload spark-assembly jar to HDFS and then set this property will 
avoid repeatedly uploading jar at runtime
 
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/deploy/spark-defaults.conf
--
diff --git a/build/deploy/spark-defaults.conf b/build/deploy/spark-defaults.conf
index 36c0ab3..78a4bc9 100644
--- a/build/deploy/spark-defaults.conf
+++ b/build/deploy/spark-defaults.conf
@@ -1,5 +1,4 @@
 spark.yarn.submit.file.replication=1
-spark.eventLog.enabled=true
 spark.yarn.max.executor.failures=3
 spark.driver.extraJavaOptions=-Dhdp.version=current
 spark.yarn.am.extraJavaOptions=-Dhdp.version=current

http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5932197..b1acbbf 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,14 +193,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
-public String getRawHdfsWorkingDirectory() {
-String root = getRequired("kylin.env.hdfs-working-dir");
-if (!root.endsWith("/")) {
-root += "/";
-}
-return root;
-}
-
 // 

 // METADATA
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 25a0801..c441618 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -63,6 +63,9 @@ abstract public class ResourceStore {
 public static final String CUBE_STATISTICS_ROOT = "/cube_statistics";
 public 

[22/39] kylin git commit: KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs

[shaofengshi]

KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6dda4fe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6dda4fe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6dda4fe

Branch: refs/heads/spark-it
Commit: f6dda4fecd5a882a17379987445a5aa0a347ff63
Parents: 4c33e38
Author: kangkaisen 
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 ++-
 .../mr/steps/FactDistinctColumnsReducer.java| 240 +++
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 175 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
 }
 }
 
+public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+@Override
+public boolean accept(Path path) {
+return path.getName().startsWith(filter);
+}
+});
+
+if (fileStatus.length == 1) {
+return fileStatus[0].getPath();
+} else {
+return null;
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/statistics";
+return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
-String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
+String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+String CFG_OUTPUT_COLUMN = "column";
+String CFG_OUTPUT_DICT = "dict";
+String CFG_OUTPUT_STATISTICS 

[02/39] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[shaofengshi]

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index e0782ce..171e299 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -16,24 +16,6 @@
  */
 package org.apache.calcite.sql2rel;
 
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
 import org.apache.calcite.avatica.util.Spaces;
 import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.plan.Convention;
@@ -85,7 +67,6 @@ import org.apache.calcite.rel.stream.Delta;
 import org.apache.calcite.rel.stream.LogicalDelta;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -116,6 +97,7 @@ import org.apache.calcite.sql.SqlCallBinding;
 import org.apache.calcite.sql.SqlDataTypeSpec;
 import org.apache.calcite.sql.SqlDelete;
 import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainFormat;
 import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.calcite.sql.SqlFunction;
 import org.apache.calcite.sql.SqlIdentifier;
@@ -177,17 +159,37 @@ import org.apache.calcite.util.NumberUtil;
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Util;
 import org.apache.calcite.util.trace.CalciteTrace;
-import org.slf4j.Logger;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+
+import org.slf4j.Logger;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
 
 /*
  * The code has synced with calcite. Hope one day, we could remove the 
hardcode override point.
@@ -209,42 +211,56 @@ import com.google.common.collect.Sets;
 public class SqlToRelConverter {
 //~ Static fields/initializers 
-
 
+protected static final Logger SQL2REL_LOGGER =
+CalciteTrace.getSqlToRelTracer();
+
+private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
 /** Size of the smallest IN list that will be converted to a semijoin to a
  * static table. */
-public static final int DEFAULT_IN_SUBQUERY_THRESHOLD = 20;
-protected static final Logger SQL2REL_LOGGER = 
CalciteTrace.getSqlToRelTracer();
-private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+
+@Deprecated // to be removed before 2.0
+public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
+DEFAULT_IN_SUB_QUERY_THRESHOLD;
 
 //~ Instance fields 

-public final SqlToRelConverter.Config config;
-public final RelOptTable.ViewExpander viewExpander;
+
 protected final SqlValidator validator;
 protected final RexBuilder rexBuilder;
 protected final Prepare.CatalogReader catalogReader;
 protected final RelOptCluster cluster;
+private DefaultValueFactory defaultValueFactory;
+private SubQueryConverter subQueryConverter;
 protected final List leaves = new ArrayList<>();
-protected final RelDataTypeFactory typeFactory;
 private final List dynamicParamSqlNodes = new 
ArrayList<>();
 

[29/39] kylin git commit: minor, upgrade default hdp version from 2.2.4.2-2 to 2.4.0.0-169

[shaofengshi]

minor, upgrade default hdp version from 2.2.4.2-2 to 2.4.0.0-169


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/571b7914
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/571b7914
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/571b7914

Branch: refs/heads/spark-it
Commit: 571b7914f763e05160fb7598bbc2bfbe54d8c7cf
Parents: b6d247c
Author: Hongbin Ma 
Authored: Tue Jan 24 15:29:05 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Jan 24 15:29:10 2017 +0800

--
 .../java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java   | 4 ++--
 .../java/org/apache/kylin/provision/BuildCubeWithEngine.java | 4 ++--
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java | 2 +-
 server/src/main/java/org/apache/kylin/rest/DebugTomcat.java  | 2 +-
 .../apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java| 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
index 03509cb..dac6aa9 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
@@ -46,7 +46,7 @@ public class DeployLocalMetaToRemoteTest {
 ClassUtil.addClasspath(new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
 System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
-throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
 }
 }
 
@@ -70,4 +70,4 @@ public class DeployLocalMetaToRemoteTest {
 System.out.println("blank");
 }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index cbec4f6..08cc6b9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -112,7 +112,7 @@ public class BuildCubeWithEngine {
 
 System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
-throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
 }
 
 
HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
@@ -363,4 +363,4 @@ public class BuildCubeWithEngine {
 }
 }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 194ea35..37ff8ae 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -274,7 +274,7 @@ public class BuildCubeWithStream {
 ClassUtil.addClasspath(new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
 System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
-throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
   

[26/39] kylin git commit: Add Zookeeper Lock

[shaofengshi]

Add Zookeeper Lock

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d3276e2e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d3276e2e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d3276e2e

Branch: refs/heads/spark-it
Commit: d3276e2e909d3001724ee8fda1304ae8b7f08c63
Parents: d23bf93
Author: xiefan46 <958034...@qq.com>
Authored: Fri Jan 20 09:48:17 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  22 +-
 .../test_case_data/sandbox/kylin.properties |   4 +
 .../storage/hdfs/ITHDFSResourceStoreTest.java   |  66 +-
 .../kylin/storage/hdfs/ITLockManagerTest.java   | 205 +++
 .../kylin/storage/hbase/HBaseResourceStore.java |   1 +
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 
 .../kylin/storage/hdfs/HDFSResourceStore.java   |  95 +++--
 .../apache/kylin/storage/hdfs/LockManager.java  | 116 +++
 .../apache/kylin/storage/hdfs/ResourceLock.java |  51 +
 10 files changed, 471 insertions(+), 175 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 44d636d..75b38ff 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,6 +193,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
+public String getRawHdfsWorkingDirectory() {
+String root = getRequired("kylin.env.hdfs-working-dir");
+if (!root.endsWith("/")) {
+root += "/";
+}
+return root;
+}
+
 // 

 // METADATA
 // 

@@ -201,11 +209,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metadata.url");
 }
 
-//for hdfs resource store
-public String getHDFSMetadataUrl() {
-return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
-}
-
 // for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);
@@ -925,4 +928,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
 }
 
+//zoo keeper
+public String getZooKeeperHost() {
+return getOptional("kylin.storage-zookeeper.host", "localhost");
+}
+
+public String getZooKeeperPort() {
+return getOptional("kylin.storage-zookeeper.port", "2181");
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index b01c377..2c2da91 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -177,3 +177,7 @@ 
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
 #kylin.engine.spark-conf.spark.yarn.queue=default
 
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
+
+
+#zoo keeper
+kylin.storage-zookeeper.host=sandbox
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index ef04957..27d8a3c 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -17,23 

[04/39] kylin git commit: minor, fix false positive warning in test output

[shaofengshi]

minor, fix false positive warning in test output


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85a1eb39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85a1eb39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85a1eb39

Branch: refs/heads/spark-it
Commit: 85a1eb399d8dcd099d4607aa802faa5ccdf4d2bf
Parents: 1e4ae54
Author: Billy Liu 
Authored: Fri Jan 20 08:13:11 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 08:13:11 2017 +0800

--
 .../org/apache/kylin/common/util/AbstractKylinTestCase.java   | 7 ++-
 .../java/org/apache/kylin/dict/lookup/SnapshotManager.java| 6 ++
 .../src/main/java/org/apache/kylin/job/dao/ExecutableDao.java | 6 ++
 .../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 2 +-
 examples/test_case_data/localmeta/cube_desc/ssb.json  | 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube1.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube2.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube3.json| 4 ++--
 .../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 4 ++--
 .../localmeta/cube_desc/test_streaming_table_cube_desc.json   | 2 +-
 examples/test_case_data/localmeta/kylin.properties| 2 +-
 examples/test_case_data/sandbox/kylin.properties  | 2 +-
 kylin-it/pom.xml  | 2 +-
 13 files changed, 33 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 2154c32..4a9804d 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -31,12 +31,17 @@ public abstract class AbstractKylinTestCase {
 public static final String[] SERVICES_WITH_CACHE = { //
 "org.apache.kylin.cube.CubeManager", //
 "org.apache.kylin.cube.CubeDescManager", //
+"org.apache.kylin.dict.lookup.SnapshotManager", //
+"org.apache.kylin.dict.DictionaryManager", //
 "org.apache.kylin.storage.hybrid.HybridManager", //
 "org.apache.kylin.metadata.realization.RealizationRegistry", //
 "org.apache.kylin.metadata.project.ProjectManager", //
 "org.apache.kylin.metadata.MetadataManager", //
+"org.apache.kylin.metadata.cachesync.Broadcaster", //
+"org.apache.kylin.metadata.badquery.BadQueryHistoryManager", //
 "org.apache.kylin.job.impl.threadpool.DistributedScheduler", //
-"org.apache.kylin.job.manager.ExecutableManager", //
+"org.apache.kylin.job.execution.ExecutableManager", //
+"org.apache.kylin.job.dao.ExecutableDao" //
 };
 
 public abstract void createTestMetadata() throws Exception;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index 085158a..b45d017 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -66,6 +66,12 @@ public class SnapshotManager {
 return r;
 }
 
+public static void clearCache() {
+synchronized (SERVICE_CACHE) {
+SERVICE_CACHE.clear();
+}
+}
+
 // 

 
 private KylinConfig config;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java 
b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index 5cae5ac..96505e6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -64,6 +64,12 @@ public class ExecutableDao {
 return r;
 }
 
+public static void clearCache() {
+

[21/39] kylin git commit: minor, refine cdh profile name

[shaofengshi]

minor, refine cdh profile name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db85d667
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db85d667
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db85d667

Branch: refs/heads/spark-it
Commit: db85d6679c5e1c067cfe2f1c58de0efb88044203
Parents: 87d1039
Author: Hongbin Ma 
Authored: Mon Jan 23 16:14:33 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 16:14:33 2017 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db85d667/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 520ac0c..bd1882e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1178,7 +1178,7 @@
 
 
 
-cdh-5.7
+cdh5.7
 
 2.6.0-cdh5.7.0
 2.6.0-cdh5.7.0

[35/39] kylin git commit: KYLIN-2414 minor rename on properties

[shaofengshi]

KYLIN-2414 minor rename on properties


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3fff6d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3fff6d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3fff6d1

Branch: refs/heads/spark-it
Commit: c3fff6d19d355e78461fa7f32d02feabc5bf63c8
Parents: e6a9382
Author: shaofengshi 
Authored: Thu Jan 26 10:53:19 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 10:53:19 2017 +0800

--
 build/conf/kylin.properties  |  8 +++-
 .../org/apache/kylin/common/KylinConfigBase.java |  4 ++--
 .../mr/steps/FactDistinctColumnsReducer.java | 19 +--
 3 files changed, 18 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 43ea17d..1232c47 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -127,9 +127,15 @@ kylin.engine.mr.max-reducer-number=500
 
 kylin.engine.mr.mapper-input-rows=100
 
+# Enable dictionary building in MR reducer
+kylin.engine.mr.build-dict-in-reducer=true
+
+# Number of reducers for fetching UHC column distinct values
+kylin.engine.mr.uhc-reducer-count=1
+
 ### CUBE | DICTIONARY ###
 
-# 'auto', 'inmem', 'layer' or 'random' for testing
+# 'auto', 'inmem' or 'layer'
 kylin.cube.algorithm=auto
 
 # A smaller threshold prefers layer, a larger threshold prefers in-mem

http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b25bcc0..6a88fc4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -740,8 +740,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
 }
 
-public boolean isReducerLocalBuildDict() {
-return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
+public boolean isBuildDictInReducerEnabled() {
+return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer", 
"true"));
 }
 
 public String getYarnStatusCheckUrl() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index cf94b30..5b795c2 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -78,7 +78,7 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
 int[] uhcIndex = 
CubeManager.getInstance(config).getUHCIndex(cubeDesc);
 int colIndex = reducerIdToColumnIndex.get(taskId);
 if (uhcIndex[colIndex] == 1)
-isReducerLocalBuildDict = false; //for UHC columns, this 
feature should be disabled
+buildDictInReducer = false; //for UHC columns, this 
feature should be disabled
 }
-if (isReducerLocalBuildDict) {
+if (buildDictInReducer) {
 builder = 
DictionaryGenerator.newDictionaryBuilder(col.getType());
 builder.init(null, 0);
 }
-logger.info("Reducer " + taskId + " handling column " + col + ", 
isReducerLocalBuildDict=" + isReducerLocalBuildDict);
+logger.info("Reducer " + taskId + " handling column " + col + ", 
buildDictInReducer=" + buildDictInReducer);
 }
 }
 
@@ -192,7 +191,7 @@ public class FactDistinctColumnsReducer extends 
KylinReducer dict = builder.build();
 outputDict(col, dict);
 }

[39/39] kylin git commit: KYLIN-2422 NumberDictionary support for decimal with extra 0 after "."

[shaofengshi]

KYLIN-2422 NumberDictionary support for decimal with extra 0 after "."


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0a3a28e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0a3a28e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0a3a28e

Branch: refs/heads/spark-it
Commit: b0a3a28eb7f5f4708790bee188071a9cec94e925
Parents: fac9f35
Author: shaofengshi 
Authored: Sat Feb 4 14:16:11 2017 +0800
Committer: shaofengshi 
Committed: Sat Feb 4 14:16:11 2017 +0800

--
 .../org/apache/kylin/dict/NumberDictionary.java | 25 
 .../apache/kylin/dict/NumberDictionaryTest.java |  5 +++-
 2 files changed, 29 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b0a3a28e/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index c55937d..de28440 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -53,6 +53,7 @@ public class NumberDictionary extends TrieDictionary {
 return;
 }
 
+
 if (len > buf.length) {
 throw new IllegalArgumentException("Too many digits for 
NumberDictionary: " + Bytes.toString(value, offset, len) + ". Internal buffer 
is only " + buf.length + " bytes");
 }
@@ -104,6 +105,30 @@ public class NumberDictionary extends TrieDictionary 
{
 
 bufOffset = start;
 bufLen = buf.length - start;
+
+// remove 0 in tail after the decimal point
+if (decimalPoint != end) {
+if (negative == true) {
+while (buf[bufOffset + bufLen - 2] == '9' && (bufOffset + 
bufLen - 2 > decimalPoint)) {
+bufLen--;
+}
+
+if (bufOffset + bufLen - 2 == decimalPoint) {
+bufLen--;
+}
+
+buf[bufOffset + bufLen - 1] = ';';
+} else {
+while (buf[bufOffset + bufLen - 1] == '0' && (bufOffset + 
bufLen - 1 > decimalPoint)) {
+bufLen--;
+}
+
+if (bufOffset + bufLen - 1 == decimalPoint) {
+bufLen--;
+}
+
+}
+}
 }
 
 int decodeNumber(byte[] returnValue, int offset) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0a3a28e/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
--
diff --git 
a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java 
b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index 1c04745..36eedf5 100644
--- 
a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ 
b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -91,11 +91,14 @@ public class NumberDictionaryTest extends 
LocalFileMetadataTestCase {
 checkCodec("-12345", "-9987654;");
 checkCodec("-12345.123", "-9987654.876;");
 checkCodec("0", "");
-checkCodec("0.0", ".0");
 //test resolved jira-1800
 checkCodec("-0.00454543543543543599877218", 
"-999.99545456456456456400122781;");
 checkCodec("-0.0099877218", 
"-999.9900122781;");
 checkCodec("12343434372493274.438403840384023840253554345345345345", 
"00012343434372493274.438403840384023840253554345345345345");
+assertEquals("0052.57", encodeNumber("52.5700"));
+assertEquals("", encodeNumber("0.00"));
+assertEquals("", encodeNumber("0.0"));
+assertEquals("-9987654.876;", 
encodeNumber("-12345.12300"));
 }
 
 private void checkCodec(String number, String code) {

[20/39] kylin git commit: KYLIN-2413 update test conf files

[shaofengshi]

KYLIN-2413 update test conf files


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87d10394
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87d10394
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87d10394

Branch: refs/heads/spark-it
Commit: 87d103948fdf8951941c9a7fd11213dd569f8aa5
Parents: eb73da7
Author: lidongsjtu 
Authored: Mon Jan 23 13:12:36 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 13:18:24 2017 +0800

--
 dev-support/sync_hbase_cdh_branches.sh  |  15 +--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |  25 
 dev-support/test_all_against_hdp_2_4_0_0_169.sh |  25 
 .../sandbox/capacity-scheduler.xml  |  17 +--
 examples/test_case_data/sandbox/core-site.xml   |  35 -
 .../test_case_data/sandbox/hadoop-policy.xml|   2 +-
 .../test_case_data/sandbox/hbase-policy.xml |   2 +-
 examples/test_case_data/sandbox/hbase-site.xml  | 100 +--
 examples/test_case_data/sandbox/hdfs-site.xml   |  84 +++-
 examples/test_case_data/sandbox/hive-site.xml   |  89 -
 examples/test_case_data/sandbox/mapred-site.xml |  57 ++---
 examples/test_case_data/sandbox/yarn-site.xml   | 127 ---
 12 files changed, 404 insertions(+), 174 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh 
b/dev-support/sync_hbase_cdh_branches.sh
index 472f402..592053a 100755
--- a/dev-support/sync_hbase_cdh_branches.sh
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -37,24 +37,17 @@ trap 'error ${LINENO} ${?}' ERR
 # 
 
 git fetch apache
-git checkout apache/$base-hbase1.x
-git format-patch -1
-git checkout apache/$base-cdh5.7
+git checkout apache/$base-hbase0.98
 git format-patch -1
 
 git checkout apache/$base
 git checkout -b tmp
 git reset apache/$base --hard
 
-git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-mvn clean compile -DskipTests
-git push apache tmp:$base-hbase1.x -f
-rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-
-git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+git am -3 --ignore-whitespace 
0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
 mvn clean compile -DskipTests
-git push apache tmp:$base-cdh5.7 -f
-rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+git push apache tmp:$base-hbase0.98 -f
+rm 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
 
 # clean up
 git checkout master

http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
deleted file mode 100755
index f7780dd..000
--- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-cd ${dir}
-cd ..
-
-mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee 
mvnverify.log

http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_4_0_0_169.sh
--
diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh 
b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
new file mode 100755
index 000..2a3d24b
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding 

[17/39] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[shaofengshi]

http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
 FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 String localCoprocessorJar;
 if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
 public static void deployCoprocessor(HTableDescriptor tableDesc) {
 try {
 initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with 
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed 
with coprocessor.");
 
 } catch (Exception ex) {
-logger.error("Error deploying coprocessor on " + 
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " + 
tableDesc.getTableName(), ex);
 logger.error("Will try creating the table without coprocessor.");
 }
 }
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
 desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
 }
 
-public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, 
Path hdfsCoprocessorJar) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 HTableDescriptor desc = 
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
 
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
 logger.info("reset coprocessor on " + tableName);
 
 logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
 
 while (desc.hasCoprocessor(CubeObserverClassOld2)) {
 desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
 desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
 }
 
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
 
 logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
 
 return true;
 }
 
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
+private static List resetCoprocessorOnHTables(final Admin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
 List processedTables = Collections.synchronizedList(new 
ArrayList());
 ExecutorService coprocessorPool = 
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
 CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
 
 private static class ResetCoprocessorWorker implements Runnable {
 private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
 private final Path hdfsCoprocessorJar;
 private final String tableName;
 private final List processedTables;
 
-public ResetCoprocessorWorker(CountDownLatch countDownLatch, 
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List 
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch 

[30/39] kylin git commit: KYLIN-2242 fix the bug that couldn’t parse partition column max/min value error

[shaofengshi]

KYLIN-2242 fix the bug that couldn’t parse partition column max/min value 
error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a86569a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a86569a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a86569a5

Branch: refs/heads/spark-it
Commit: a86569a5c0b49f5a607f642a70af7a826fda3edf
Parents: 571b791
Author: shaofengshi 
Authored: Tue Jan 24 15:59:19 2017 +0800
Committer: shaofengshi 
Committed: Tue Jan 24 15:59:38 2017 +0800

--
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  | 20 +++-
 1 file changed, 15 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a86569a5/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 79fe657..2efd718 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -18,7 +18,9 @@
 
 package org.apache.kylin.engine.mr.steps;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -79,6 +81,10 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 
 private void updateTimeRange(CubeSegment segment) throws IOException {
 final TblColRef partitionCol = 
segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
+
+if (partitionCol == null) {
+return;
+}
 final String factColumnsInputPath = 
this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
 Path colDir = new Path(factColumnsInputPath, 
partitionCol.getIdentity());
 FileSystem fs = HadoopUtil.getWorkingFileSystem();
@@ -88,15 +94,19 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 }
 
 FSDataInputStream is = null;
-long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
+BufferedReader bufferedReader = null;
+InputStreamReader isr = null;
+long minValue, maxValue;
 try {
 is = fs.open(outputFile);
-long min = is.readLong();
-long max = is.readLong();
-minValue = Math.min(min, minValue);
-maxValue = Math.max(max, maxValue);
+isr = new InputStreamReader(is);
+bufferedReader = new BufferedReader(isr);
+minValue = Long.parseLong(bufferedReader.readLine());
+maxValue = Long.parseLong(bufferedReader.readLine());
 } finally {
 IOUtils.closeQuietly(is);
+IOUtils.closeQuietly(isr);
+IOUtils.closeQuietly(bufferedReader);
 }
 logger.info("updateTimeRange step. minValue:" + minValue + " 
maxValue:" + maxValue);
 segment.setDateRangeStart(minValue);

[05/39] kylin git commit: KYLIN-2394 merge CALCITE-1592

[shaofengshi]

KYLIN-2394 merge CALCITE-1592


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34f2d3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34f2d3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34f2d3e

Branch: refs/heads/spark-it
Commit: d34f2d3e2bf691c55077a9a76a89217ae915bbdd
Parents: dfb5fac
Author: Billy Liu 
Authored: Fri Jan 20 15:42:37 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 15:42:37 2017 +0800

--
 .../calcite/sql2rel/SqlToRelConverter.java  | 51 +---
 1 file changed, 23 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d34f2d3e/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 171e299..cc9c9a1 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -935,19 +935,21 @@ public class SqlToRelConverter {
 /**
  * Push down all the NOT logical operators into any IN/NOT IN operators.
  *
+ * @param scope Scope where {@code sqlNode} occurs
  * @param sqlNode the root node from which to look for NOT operators
  * @return the transformed SqlNode representation with NOT pushed down.
  */
-private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+private static SqlNode pushDownNotForIn(SqlValidatorScope scope,
+SqlNode sqlNode) {
 if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
 SqlCall sqlCall = (SqlCall) sqlNode;
 if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
 || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
 SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
 for (int i = 0; i < sqlOperands.length; i++) {
-sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+sqlOperands[i] = pushDownNotForIn(scope, sqlOperands[i]);
 }
-return sqlNode;
+return reg(scope, sqlNode);
 } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
 SqlNode childNode = sqlCall.operand(0);
 assert childNode instanceof SqlCall;
@@ -956,48 +958,34 @@ public class SqlToRelConverter {
 SqlNode[] andOperands = childSqlCall.getOperands();
 SqlNode[] orOperands = new SqlNode[andOperands.length];
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-andOperands[i]);
+orOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]));
 }
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] = pushDownNotForIn(orOperands[i]);
+orOperands[i] = pushDownNotForIn(scope, orOperands[i]);
 }
-return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
-orOperands[0], orOperands[1]);
+return reg(scope, 
SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], 
orOperands[1]));
 } else if (childSqlCall.getOperator() == 
SqlStdOperatorTable.OR) {
 SqlNode[] orOperands = childSqlCall.getOperands();
 SqlNode[] andOperands = new SqlNode[orOperands.length];
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-orOperands[i]);
+andOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]));
 }
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] = pushDownNotForIn(andOperands[i]);
+andOperands[i] = pushDownNotForIn(scope, 
andOperands[i]);
 }
-return 
SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
-   

[09/39] kylin git commit: KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs

[shaofengshi]

KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7de8aa12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7de8aa12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7de8aa12

Branch: refs/heads/spark-it
Commit: 7de8aa1203a72bad105ed692f7100535939b03af
Parents: c2229c9
Author: kangkaisen 
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: kangkaisen 
Committed: Sat Jan 21 23:19:50 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 ++-
 .../mr/steps/FactDistinctColumnsReducer.java| 240 +++
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 175 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
 }
 }
 
+public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+@Override
+public boolean accept(Path path) {
+return path.getName().startsWith(filter);
+}
+});
+
+if (fileStatus.length == 1) {
+return fileStatus[0].getPath();
+} else {
+return null;
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/statistics";
+return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
-String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
+String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+String CFG_OUTPUT_COLUMN = "column";
+String CFG_OUTPUT_DICT = "dict";
+String CFG_OUTPUT_STATISTICS = 

[25/39] kylin git commit: KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase

[shaofengshi]

KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d23bf930
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d23bf930
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d23bf930

Branch: refs/heads/spark-it
Commit: d23bf930da0b542d0e6981917e6bde055839577a
Parents: db85d66
Author: xiefan46 <958034...@qq.com>
Authored: Wed Jan 11 10:00:19 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   5 +
 .../common/persistence/ResourceStoreTest.java   |   3 +-
 .../test_case_data/sandbox/kylin.properties |   1 +
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 117 +++
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 +
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 198 +++
 7 files changed, 409 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 05df177..44d636d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -201,6 +201,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metadata.url");
 }
 
+//for hdfs resource store
+public String getHDFSMetadataUrl() {
+return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
+}
+
 // for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);

http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 4c31a15..ddaf481 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -110,9 +110,10 @@ public class ResourceStoreTest {
 }
 
 // list
-NavigableSet list;
+NavigableSet list = null;
 
 list = store.listResources(dir1);
+System.out.println(list);
 assertTrue(list.contains(path1));
 assertTrue(list.contains(path2) == false);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 6c512dc..b01c377 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -41,6 +41,7 @@ kylin.source.hive.client=cli
 # The metadata store in hbase
 kylin.metadata.url=kylin_default_instance@hbase
 
+
 # The storage for final cube file in hbase
 kylin.storage.url=hbase
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
new file mode 100644
index 000..ef04957
--- /dev/null
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under 

[11/39] kylin git commit: KYLIN-2412 Unclosed DataOutputStream in RoaringBitmapCounter#write()

[shaofengshi]

KYLIN-2412 Unclosed DataOutputStream in RoaringBitmapCounter#write()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d264339b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d264339b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d264339b

Branch: refs/heads/spark-it
Commit: d264339b1c16c195ffafc2217b793d81bdbd6434
Parents: 7576a09
Author: gaodayue 
Authored: Sun Jan 22 14:02:09 2017 +0800
Committer: gaodayue 
Committed: Sun Jan 22 14:02:09 2017 +0800

--
 .../org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java| 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d264339b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index cd07d20..fb9dcfc 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -107,7 +107,9 @@ public class RoaringBitmapCounter implements BitmapCounter {
 if (out.remaining() < bitmap.serializedSizeInBytes()) {
 throw new BufferOverflowException();
 }
-bitmap.serialize(new DataOutputStream(new 
ByteBufferOutputStream(out)));
+try (DataOutputStream dos = new DataOutputStream(new 
ByteBufferOutputStream(out))) {
+bitmap.serialize(dos);
+}
 }
 
 @Override

[12/39] kylin git commit: Merge branch 'KYLIN-2394'

[shaofengshi]

Merge branch 'KYLIN-2394'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c4591a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c4591a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c4591a6

Branch: refs/heads/spark-it
Commit: 2c4591a609c8d8141c880722a3b8b282b785133e
Parents: d264339 d34f2d3
Author: Billy Liu 
Authored: Sun Jan 22 17:43:16 2017 +0800
Committer: Billy Liu 
Committed: Sun Jan 22 17:43:16 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3782 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2348 insertions(+), 1494 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2c4591a6/kylin-it/pom.xml
--

[38/39] kylin git commit: KYLIN-2421 Add spark engine to Integration Test

[shaofengshi]

KYLIN-2421 Add spark engine to Integration Test


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fac9f358
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fac9f358
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fac9f358

Branch: refs/heads/spark-it
Commit: fac9f358c374d6ce92a2cf04c72ccda9049c27cb
Parents: 546f88f
Author: shaofengshi 
Authored: Fri Jan 20 11:28:57 2017 +0800
Committer: shaofengshi 
Committed: Sat Feb 4 14:13:44 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  10 ++
 .../measure/bitmap/RoaringBitmapCounter.java|   3 +-
 .../bitmap/RoaringBitmapCounterFactory.java |   3 +-
 .../measure/percentile/PercentileCounter.java   |  22 ++-
 .../percentile/PercentileSerializer.java|   6 +-
 .../kylin/measure/topn/TopNAggregator.java  |   5 +-
 .../percentile/PercentileCounterTest.java   |  47 ++
 .../kylin/engine/mr/BatchCubingJobBuilder2.java |   8 +-
 .../engine/spark/KylinKryoRegistrator.java  | 161 +++
 .../spark/SparkBatchCubingJobBuilder2.java  |  12 +-
 .../apache/kylin/engine/spark/SparkCubing.java  | 123 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |  65 
 .../localmeta/cube_desc/ci_inner_join_cube.json |  14 +-
 examples/test_case_data/sandbox/core-site.xml   |   2 +
 .../test_case_data/sandbox/kylin.properties |  29 ++--
 kylin-it/pom.xml|  21 +++
 .../kylin/provision/BuildCubeWithEngine.java|  25 +++
 17 files changed, 355 insertions(+), 201 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fac9f358/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6a88fc4..fe15b1e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -68,6 +68,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 return sparkHome;
 }
 
+sparkHome = System.getProperty("SPARK_HOME");
+if (StringUtils.isNotEmpty(sparkHome)) {
+logger.info("SPARK_HOME was set to " + sparkHome);
+return sparkHome;
+}
+
 return getKylinHome() + File.separator + "spark";
 }
 
@@ -760,6 +766,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.engine.spark.env.hadoop-conf-dir", "");
 }
 
+public void setHadoopConfDir(String hadoopConfDir) {
+setProperty("kylin.engine.spark.env.hadoop-conf-dir", hadoopConfDir);
+}
+
 public String getSparkAdditionalJars() {
 return getOptional("kylin.engine.spark.additional-jars", "");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fac9f358/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index fb9dcfc..eec45f2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -24,6 +24,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.Serializable;
 import java.nio.BufferOverflowException;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
@@ -31,7 +32,7 @@ import java.util.Iterator;
 /**
  * A {@link BitmapCounter} based on roaring bitmap.
  */
-public class RoaringBitmapCounter implements BitmapCounter {
+public class RoaringBitmapCounter implements BitmapCounter, Serializable {
 
 private ImmutableRoaringBitmap bitmap;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/fac9f358/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
index a71df95..822afa2 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java
+++ 

[24/39] kylin git commit: Add logging info

[shaofengshi]

Add logging info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/971bf800
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/971bf800
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/971bf800

Branch: refs/heads/spark-it
Commit: 971bf80009c446a873ddc0927151eb1e68d36e27
Parents: e217ae8
Author: shaofengshi 
Authored: Mon Jan 23 11:49:27 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../org/apache/kylin/source/kafka/job/MergeOffsetStep.java  | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/971bf800/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
index 9cadd72..914fca2 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/MergeOffsetStep.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.base.Preconditions;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
@@ -38,6 +39,7 @@ import org.slf4j.LoggerFactory;
 public class MergeOffsetStep extends AbstractExecutable {
 
 private static final Logger logger = 
LoggerFactory.getLogger(MergeOffsetStep.class);
+
 public MergeOffsetStep() {
 super();
 }
@@ -46,12 +48,15 @@ public class MergeOffsetStep extends AbstractExecutable {
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
 final CubeManager cubeManager = 
CubeManager.getInstance(context.getConfig());
 final CubeInstance cube = 
cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
-final CubeSegment segment = 
cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
+final String segmentId = 
CubingExecutableUtil.getSegmentId(this.getParams());
+final CubeSegment segment = cube.getSegmentById(segmentId);
 
+Preconditions.checkNotNull(segment, "Cube segment '" + segmentId + "' 
not found.");
 List mergingSegs = cube.getMergingSegments(segment);
 
-Collections.sort(mergingSegs);
+Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment 
not exist.");
 
+Collections.sort(mergingSegs);
 final CubeSegment first = mergingSegs.get(0);
 final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1);
 

[34/39] kylin git commit: KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217

[shaofengshi]

KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6a9382b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6a9382b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6a9382b

Branch: refs/heads/spark-it
Commit: e6a9382b8d357e5bb13be2f678ebabb8bad75dc7
Parents: 5eae37e
Author: xiefan46 <958034...@qq.com>
Authored: Tue Jan 24 11:14:40 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 09:56:54 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 3 ---
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java  | 6 ++
 2 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b1acbbf..b25bcc0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -741,9 +741,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public boolean isReducerLocalBuildDict() {
-if (getUHCReducerCount() != 1) {
-return false;
-}
 return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8c56bdf..cf94b30 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -132,6 +132,12 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
+int[] uhcIndex = 
CubeManager.getInstance(config).getUHCIndex(cubeDesc);
+int colIndex = reducerIdToColumnIndex.get(taskId);
+if (uhcIndex[colIndex] == 1)
+isReducerLocalBuildDict = false; //for UHC columns, this 
feature should be disabled
+}
 if (isReducerLocalBuildDict) {
 builder = 
DictionaryGenerator.newDictionaryBuilder(col.getType());
 builder.init(null, 0);

kylin git commit: KYLIN-2420 Record zookeeper lock holder information

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 7611338b5 -> 56c7aa505


KYLIN-2420 Record zookeeper lock holder information

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56c7aa50
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56c7aa50
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56c7aa50

Branch: refs/heads/master
Commit: 56c7aa505706f9aa878b4d3804cdf3f357a7931f
Parents: 7611338
Author: Roger Shi 
Authored: Sat Feb 4 13:43:03 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 13:46:59 2017 +0800

--
 .../storage/hbase/util/ZookeeperJobLock.java| 26 +++-
 1 file changed, 25 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56c7aa50/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
index bdd3981..7bf7498 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.storage.hbase.util;
 
+import java.lang.management.ManagementFactory;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
 
@@ -35,6 +37,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.job.lock.JobLock;
 import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,9 +66,14 @@ public class ZookeeperJobLock implements JobLock {
 }
 
 RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
-this.zkClient = CuratorFrameworkFactory.newClient(zkConnectString, 
retryPolicy);
+try {
+this.zkClient = 
CuratorFrameworkFactory.builder().connectString(zkConnectString).retryPolicy(retryPolicy).defaultData(getIpProcess()).build();
+} catch (UnknownHostException e) {
+throw new RuntimeException(e);
+}
 this.zkClient.start();
 this.sharedLock = new InterProcessMutex(zkClient, this.scheduleID);
+
 boolean hasLock = false;
 try {
 hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
@@ -74,9 +82,20 @@ public class ZookeeperJobLock implements JobLock {
 }
 if (!hasLock) {
 logger.warn("fail to acquire lock, scheduler has not been started; 
maybe another kylin process is still running?");
+try {
+for (String node : sharedLock.getParticipantNodes()) {
+logger.warn("lock holder info: {}", new 
String(zkClient.getData().forPath(node)));
+}
+} catch (Exception e) {
+logger.warn("error check participant", e);
+if (!(e instanceof KeeperException.NoNodeException)) {
+throw new RuntimeException(e);
+}
+}
 zkClient.close();
 return false;
 }
+
 return true;
 }
 
@@ -115,4 +134,9 @@ public class ZookeeperJobLock implements JobLock {
 private String schedulerId() {
 return ZOOKEEPER_LOCK_PATH + "/" + 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
 }
+
+private byte[] getIpProcess() throws UnknownHostException {
+logger.info("get IP and processId: {}", 
ManagementFactory.getRuntimeMXBean().getName().getBytes());
+return ManagementFactory.getRuntimeMXBean().getName().getBytes();
+}
 }

[45/47] kylin git commit: minor, stablize DefaultSchedulerTest

[billyliu]

minor, stablize DefaultSchedulerTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a058bfb8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a058bfb8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a058bfb8

Branch: refs/heads/KYLIN-2361
Commit: a058bfb8b0490fe36b7fe4da026028411ed208a5
Parents: 546f88f
Author: Li Yang 
Authored: Fri Feb 3 13:43:56 2017 +0800
Committer: Li Yang 
Committed: Fri Feb 3 13:43:56 2017 +0800

--
 .../job/impl/threadpool/BaseSchedulerTest.java  | 36 ++--
 1 file changed, 26 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a058bfb8/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index fdf5252..1ada9a1 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -31,11 +31,15 @@ import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.lock.MockJobLock;
 import org.junit.After;
 import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  */
 public abstract class BaseSchedulerTest extends LocalFileMetadataTestCase {
 
+private static final Logger logger = 
LoggerFactory.getLogger(BaseSchedulerTest.class);
+
 private DefaultScheduler scheduler;
 
 protected ExecutableManager jobService;
@@ -70,19 +74,31 @@ public abstract class BaseSchedulerTest extends 
LocalFileMetadataTestCase {
 }
 
 protected void waitForJobFinish(String jobId) {
-while (true) {
-AbstractExecutable job = jobService.getJob(jobId);
-final ExecutableState status = job.getStatus();
-if (status == ExecutableState.SUCCEED || status == 
ExecutableState.ERROR || status == ExecutableState.STOPPED || status == 
ExecutableState.DISCARDED) {
-break;
-} else {
-try {
-Thread.sleep(5000);
-} catch (InterruptedException e) {
-e.printStackTrace();
+int error = 0;
+final int errorLimit = 3;
+
+while (error < errorLimit) {
+try {
+Thread.sleep(2000);
+} catch (InterruptedException e) {
+e.printStackTrace();
+}
+
+try {
+AbstractExecutable job = jobService.getJob(jobId);
+ExecutableState status = job.getStatus();
+if (status == ExecutableState.SUCCEED || status == 
ExecutableState.ERROR || status == ExecutableState.STOPPED || status == 
ExecutableState.DISCARDED) {
+break;
 }
+} catch (Exception ex) {
+logger.error("", ex);
+error++;
 }
 }
+
+if (error >= errorLimit) {
+throw new RuntimeException("waitForJobFinish() encounters 
exceptions, see logs above");
+}
 }
 
 protected void waitForJobStatus(String jobId, ExecutableState state, long 
interval) {

[09/47] kylin git commit: KYLIN-2411 Kill MR job on pause

[billyliu]

KYLIN-2411 Kill MR job on pause


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b49c9e39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b49c9e39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b49c9e39

Branch: refs/heads/KYLIN-2361
Commit: b49c9e3951c29a21e71f179ee77e2295fbc52ba9
Parents: 38c3e7b
Author: shaofengshi 
Authored: Thu Jan 19 11:02:12 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:37:42 2017 +0800

--
 .../org/apache/kylin/job/execution/ExecutableManager.java| 2 +-
 .../apache/kylin/engine/mr/common/MapReduceExecutable.java   | 8 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 466cdad..48cedb5 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -347,7 +347,7 @@ public class ExecutableManager {
 for (AbstractExecutable task : tasks) {
 if (task.getId().compareTo(stepId) >= 0) {
 logger.debug("rollback task : " + task);
-updateJobOutput(task.getId(), ExecutableState.READY, null, 
null);
+updateJobOutput(task.getId(), ExecutableState.READY, 
Maps.newHashMap(), "");
 }
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index f887c4c..6de07ca 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -148,7 +148,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 //boolean useKerberosAuth = 
context.getConfig().isGetJobStatusWithKerberos();
 //HadoopStatusChecker statusChecker = new 
HadoopStatusChecker(restStatusCheckUrl, mrJobId, output, useKerberosAuth);
 JobStepStatusEnum status = JobStepStatusEnum.NEW;
-while (!isDiscarded()) {
+while (!isDiscarded() && !isPaused()) {
 
 JobStepStatusEnum newStatus = 
HadoopJobStatusChecker.checkStatus(job, output);
 if (status == JobStepStatusEnum.KILLED) {
@@ -184,7 +184,11 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 }
 }
 
-return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
+if (isDiscarded()) {
+return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
+} else {
+return new ExecuteResult(ExecuteResult.State.STOPPED, 
output.toString());
+}
 
 } catch (ReflectiveOperationException e) {
 logger.error("error getMapReduceJobClass, class name:" + 
getParam(KEY_MR_JOB), e);

[40/47] kylin git commit: KYLIN-2374 code review

[billyliu]

KYLIN-2374 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eae37ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eae37ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eae37ef

Branch: refs/heads/KYLIN-2361
Commit: 5eae37ef18ca51027c6bb2cfd3410fefc7982f2a
Parents: a2a59c4
Author: shaofengshi 
Authored: Thu Jan 26 09:55:48 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 09:55:48 2017 +0800

--
 build/conf/kylin.properties |  3 +-
 build/deploy/spark-defaults.conf|  1 -
 .../apache/kylin/common/KylinConfigBase.java|  8 --
 .../kylin/common/persistence/ResourceStore.java |  3 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |  2 +-
 .../ExtendedColumnMeasureType.java  |  8 +-
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 36 +++-
 .../kylin/storage/hbase/HBaseResourceStore.java |  3 +-
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 90 +++-
 9 files changed, 97 insertions(+), 57 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index eceb886..43ea17d 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -211,8 +211,9 @@ kylin.engine.spark-conf.spark.executor.memory=4G
 kylin.engine.spark-conf.spark.executor.cores=4
 kylin.engine.spark-conf.spark.executor.instances=8
 kylin.engine.spark-conf.spark.storage.memoryFraction=0.3
-kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.eventLog.enabled=true
 kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
 ## manually upload spark-assembly jar to HDFS and then set this property will 
avoid repeatedly uploading jar at runtime
 
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/deploy/spark-defaults.conf
--
diff --git a/build/deploy/spark-defaults.conf b/build/deploy/spark-defaults.conf
index 36c0ab3..78a4bc9 100644
--- a/build/deploy/spark-defaults.conf
+++ b/build/deploy/spark-defaults.conf
@@ -1,5 +1,4 @@
 spark.yarn.submit.file.replication=1
-spark.eventLog.enabled=true
 spark.yarn.max.executor.failures=3
 spark.driver.extraJavaOptions=-Dhdp.version=current
 spark.yarn.am.extraJavaOptions=-Dhdp.version=current

http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5932197..b1acbbf 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,14 +193,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
-public String getRawHdfsWorkingDirectory() {
-String root = getRequired("kylin.env.hdfs-working-dir");
-if (!root.endsWith("/")) {
-root += "/";
-}
-return root;
-}
-
 // 

 // METADATA
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 25a0801..c441618 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -63,6 +63,9 @@ abstract public class ResourceStore {
 public static final String CUBE_STATISTICS_ROOT = "/cube_statistics";
 public 

[36/47] kylin git commit: minor, upgrade default hdp version from 2.2.4.2-2 to 2.4.0.0-169

[billyliu]

minor, upgrade default hdp version from 2.2.4.2-2 to 2.4.0.0-169


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/571b7914
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/571b7914
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/571b7914

Branch: refs/heads/KYLIN-2361
Commit: 571b7914f763e05160fb7598bbc2bfbe54d8c7cf
Parents: b6d247c
Author: Hongbin Ma 
Authored: Tue Jan 24 15:29:05 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Jan 24 15:29:10 2017 +0800

--
 .../java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java   | 4 ++--
 .../java/org/apache/kylin/provision/BuildCubeWithEngine.java | 4 ++--
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java | 2 +-
 server/src/main/java/org/apache/kylin/rest/DebugTomcat.java  | 2 +-
 .../apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java| 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
index 03509cb..dac6aa9 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
@@ -46,7 +46,7 @@ public class DeployLocalMetaToRemoteTest {
 ClassUtil.addClasspath(new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
 System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
-throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
 }
 }
 
@@ -70,4 +70,4 @@ public class DeployLocalMetaToRemoteTest {
 System.out.println("blank");
 }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index cbec4f6..08cc6b9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -112,7 +112,7 @@ public class BuildCubeWithEngine {
 
 System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
-throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
 }
 
 
HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
@@ -363,4 +363,4 @@ public class BuildCubeWithEngine {
 }
 }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 194ea35..37ff8ae 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -274,7 +274,7 @@ public class BuildCubeWithStream {
 ClassUtil.addClasspath(new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
 System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
-throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
 

[32/47] kylin git commit: KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase

[billyliu]

KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d23bf930
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d23bf930
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d23bf930

Branch: refs/heads/KYLIN-2361
Commit: d23bf930da0b542d0e6981917e6bde055839577a
Parents: db85d66
Author: xiefan46 <958034...@qq.com>
Authored: Wed Jan 11 10:00:19 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   5 +
 .../common/persistence/ResourceStoreTest.java   |   3 +-
 .../test_case_data/sandbox/kylin.properties |   1 +
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 117 +++
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 +
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 198 +++
 7 files changed, 409 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 05df177..44d636d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -201,6 +201,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metadata.url");
 }
 
+//for hdfs resource store
+public String getHDFSMetadataUrl() {
+return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
+}
+
 // for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);

http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 4c31a15..ddaf481 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -110,9 +110,10 @@ public class ResourceStoreTest {
 }
 
 // list
-NavigableSet list;
+NavigableSet list = null;
 
 list = store.listResources(dir1);
+System.out.println(list);
 assertTrue(list.contains(path1));
 assertTrue(list.contains(path2) == false);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 6c512dc..b01c377 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -41,6 +41,7 @@ kylin.source.hive.client=cli
 # The metadata store in hbase
 kylin.metadata.url=kylin_default_instance@hbase
 
+
 # The storage for final cube file in hbase
 kylin.storage.url=hbase
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
new file mode 100644
index 000..ef04957
--- /dev/null
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under 

[24/47] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
 FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 String localCoprocessorJar;
 if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
 public static void deployCoprocessor(HTableDescriptor tableDesc) {
 try {
 initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with 
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed 
with coprocessor.");
 
 } catch (Exception ex) {
-logger.error("Error deploying coprocessor on " + 
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " + 
tableDesc.getTableName(), ex);
 logger.error("Will try creating the table without coprocessor.");
 }
 }
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
 desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
 }
 
-public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, 
Path hdfsCoprocessorJar) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 HTableDescriptor desc = 
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
 
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
 logger.info("reset coprocessor on " + tableName);
 
 logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
 
 while (desc.hasCoprocessor(CubeObserverClassOld2)) {
 desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
 desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
 }
 
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
 
 logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
 
 return true;
 }
 
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
+private static List resetCoprocessorOnHTables(final Admin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
 List processedTables = Collections.synchronizedList(new 
ArrayList());
 ExecutorService coprocessorPool = 
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
 CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
 
 private static class ResetCoprocessorWorker implements Runnable {
 private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
 private final Path hdfsCoprocessorJar;
 private final String tableName;
 private final List processedTables;
 
-public ResetCoprocessorWorker(CountDownLatch countDownLatch, 
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List 
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch 

[25/47] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/169c3de6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/169c3de6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/169c3de6

Branch: refs/heads/KYLIN-2361
Commit: 169c3de6456f394d049e9550d65a98f6aa0a316f
Parents: 68e87f0
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 13:01:20 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index e02bf19..cbec4f6 100644
--- 

[42/47] kylin git commit: KYLIN-2414 minor rename on properties

[billyliu]

KYLIN-2414 minor rename on properties


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3fff6d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3fff6d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3fff6d1

Branch: refs/heads/KYLIN-2361
Commit: c3fff6d19d355e78461fa7f32d02feabc5bf63c8
Parents: e6a9382
Author: shaofengshi 
Authored: Thu Jan 26 10:53:19 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 10:53:19 2017 +0800

--
 build/conf/kylin.properties  |  8 +++-
 .../org/apache/kylin/common/KylinConfigBase.java |  4 ++--
 .../mr/steps/FactDistinctColumnsReducer.java | 19 +--
 3 files changed, 18 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 43ea17d..1232c47 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -127,9 +127,15 @@ kylin.engine.mr.max-reducer-number=500
 
 kylin.engine.mr.mapper-input-rows=100
 
+# Enable dictionary building in MR reducer
+kylin.engine.mr.build-dict-in-reducer=true
+
+# Number of reducers for fetching UHC column distinct values
+kylin.engine.mr.uhc-reducer-count=1
+
 ### CUBE | DICTIONARY ###
 
-# 'auto', 'inmem', 'layer' or 'random' for testing
+# 'auto', 'inmem' or 'layer'
 kylin.cube.algorithm=auto
 
 # A smaller threshold prefers layer, a larger threshold prefers in-mem

http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b25bcc0..6a88fc4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -740,8 +740,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
 }
 
-public boolean isReducerLocalBuildDict() {
-return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
+public boolean isBuildDictInReducerEnabled() {
+return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer", 
"true"));
 }
 
 public String getYarnStatusCheckUrl() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index cf94b30..5b795c2 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -78,7 +78,7 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
 int[] uhcIndex = 
CubeManager.getInstance(config).getUHCIndex(cubeDesc);
 int colIndex = reducerIdToColumnIndex.get(taskId);
 if (uhcIndex[colIndex] == 1)
-isReducerLocalBuildDict = false; //for UHC columns, this 
feature should be disabled
+buildDictInReducer = false; //for UHC columns, this 
feature should be disabled
 }
-if (isReducerLocalBuildDict) {
+if (buildDictInReducer) {
 builder = 
DictionaryGenerator.newDictionaryBuilder(col.getType());
 builder.init(null, 0);
 }
-logger.info("Reducer " + taskId + " handling column " + col + ", 
isReducerLocalBuildDict=" + isReducerLocalBuildDict);
+logger.info("Reducer " + taskId + " handling column " + col + ", 
buildDictInReducer=" + buildDictInReducer);
 }
 }
 
@@ -192,7 +191,7 @@ public class FactDistinctColumnsReducer extends 
KylinReducer dict = builder.build();
 outputDict(col, dict);
 }

[21/47] kylin git commit: KYLIN 2396 GUI for Percentile pre aggregation implementation

[billyliu]

KYLIN 2396 GUI for Percentile pre aggregation implementation

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b56bc622
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b56bc622
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b56bc622

Branch: refs/heads/KYLIN-2361
Commit: b56bc622086495f7d0047d01af57efc919b2ecce
Parents: 28673a8
Author: luguosheng <550175...@qq.com>
Authored: Mon Jan 23 11:24:39 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 11:32:46 2017 +0800

--
 webapp/app/js/controllers/cubeMeasures.js  | 2 ++
 webapp/app/partials/cubeDesigner/measures.html | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 7c155fe..91c38f2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -355,6 +355,8 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   $scope.newMeasure.function.parameter.type= 'column';
   $scope.newMeasure.function.returntype = "extendedcolumn(100)";
   return;
+}else if($scope.newMeasure.function.expression=='PERCENTILE'){
+  $scope.newMeasure.function.parameter.type= 'column';
 }else{
   $scope.nextParameterInit();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index 4a48843..b7475b6 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -116,7 +116,7 @@
   
   Param Type
   
-  
   
   constant
+ ng-if="newMeasure.function.expression 
== 'COUNT'||newMeasure.function.expression == 'PERCENTILE'"
+ 
>{{newMeasure.function.parameter.type}}
   
   
   

[37/47] kylin git commit: KYLIN-2242 fix the bug that couldn’t parse partition column max/min value error

[billyliu]

KYLIN-2242 fix the bug that couldn’t parse partition column max/min value 
error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a86569a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a86569a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a86569a5

Branch: refs/heads/KYLIN-2361
Commit: a86569a5c0b49f5a607f642a70af7a826fda3edf
Parents: 571b791
Author: shaofengshi 
Authored: Tue Jan 24 15:59:19 2017 +0800
Committer: shaofengshi 
Committed: Tue Jan 24 15:59:38 2017 +0800

--
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  | 20 +++-
 1 file changed, 15 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a86569a5/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 79fe657..2efd718 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -18,7 +18,9 @@
 
 package org.apache.kylin.engine.mr.steps;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -79,6 +81,10 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 
 private void updateTimeRange(CubeSegment segment) throws IOException {
 final TblColRef partitionCol = 
segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
+
+if (partitionCol == null) {
+return;
+}
 final String factColumnsInputPath = 
this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
 Path colDir = new Path(factColumnsInputPath, 
partitionCol.getIdentity());
 FileSystem fs = HadoopUtil.getWorkingFileSystem();
@@ -88,15 +94,19 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 }
 
 FSDataInputStream is = null;
-long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
+BufferedReader bufferedReader = null;
+InputStreamReader isr = null;
+long minValue, maxValue;
 try {
 is = fs.open(outputFile);
-long min = is.readLong();
-long max = is.readLong();
-minValue = Math.min(min, minValue);
-maxValue = Math.max(max, maxValue);
+isr = new InputStreamReader(is);
+bufferedReader = new BufferedReader(isr);
+minValue = Long.parseLong(bufferedReader.readLine());
+maxValue = Long.parseLong(bufferedReader.readLine());
 } finally {
 IOUtils.closeQuietly(is);
+IOUtils.closeQuietly(isr);
+IOUtils.closeQuietly(bufferedReader);
 }
 logger.info("updateTimeRange step. minValue:" + minValue + " 
maxValue:" + maxValue);
 segment.setDateRangeStart(minValue);

[14/47] kylin git commit: KYLIN-2308 Allow user to set more columnFamily in web

[billyliu]

KYLIN-2308 Allow user to set more columnFamily in web

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8331d8d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8331d8d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8331d8d0

Branch: refs/heads/KYLIN-2361
Commit: 8331d8d0ad37d24bdbfa90d1032bcdd063aa0a59
Parents: a853a7c
Author: kangkaisen 
Authored: Sat Jan 7 15:34:57 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 17:25:28 2017 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 75 +++-
 webapp/app/js/controllers/cubeEdit.js   | 51 ---
 webapp/app/js/controllers/cubeSchema.js | 13 +++
 webapp/app/js/filters/filter.js | 22 +++--
 .../cubeDesigner/advanced_settings.html | 92 +++-
 5 files changed, 194 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 760133a..39d36b0 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -278,11 +278,84 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 $scope.isReuse=!$scope.isReuse;
   }
 
-  $scope.removeDictionaries =  function(arr,element){
+  $scope.removeElement =  function(arr,element){
 var index = arr.indexOf(element);
 if (index > -1) {
   arr.splice(index, 1);
 }
   };
 
+  $scope.newColFamily = function (index) {
+return {
+"name": "F" + index,
+"columns": [
+  {
+"qualifier": "M",
+"measure_refs": []
+  }
+]
+  };
+  };
+
+  $scope.initColumnFamily = function () {
+$scope.cubeMetaFrame.hbase_mapping.column_family = [];
+var normalMeasures = [], distinctCountMeasures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  if (measure.function.expression === 'COUNT_DISTINCT') {
+distinctCountMeasures.push(measure);
+  } else {
+normalMeasures.push(measure);
+  }
+});
+if (normalMeasures.length > 0) {
+  var nmcf = $scope.newColFamily(1);
+  angular.forEach(normalMeasures, function (normalM, index) {
+nmcf.columns[0].measure_refs.push(normalM.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(nmcf);
+}
+
+if (distinctCountMeasures.length > 0) {
+  var dccf = $scope.newColFamily(2);
+  angular.forEach(distinctCountMeasures, function (dcm, index) {
+dccf.columns[0].measure_refs.push(dcm.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(dccf);
+}
+  };
+
+  $scope.getAllMeasureNames = function () {
+var measures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  measures.push(measure.name);
+});
+return measures;
+  };
+
+  $scope.getAssignedMeasureNames = function () {
+var assignedMeasures = [];
+angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function 
(colFamily, index) {
+  angular.forEach(colFamily.columns[0].measure_refs, function (measure, 
index) {
+assignedMeasures.push(measure);
+  });
+});
+return assignedMeasures;
+  };
+
+  if ($scope.getAllMeasureNames().length != 
$scope.getAssignedMeasureNames().length) {
+$scope.initColumnFamily();
+  }
+
+
+  $scope.addColumnFamily = function () {
+var colFamily = 
$scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 
1);
+$scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
+  };
+
+  $scope.refreshColumnFamily = function (column_familys, index, colFamily) {
+if (column_familys) {
+  column_familys[index] = colFamily;
+}
+  };
+
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index edbb421..da19b22 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -284,25 +284,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return type;
   };
 
-  var ColFamily = function () {
-var index = 1;
-return function () {
-  var newColFamily =
-  {
-"name": "f" + index,
-"columns": [
-  {
-

[04/47] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index cd784f4..874ead6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -66,13 +66,15 @@
 
 
 
false
+
 
-
 
-
com.fasterxml.jackson.core:*
+com.google.protobuf:*
+commons-logging:*
+commons-codec:*
 
 
+
 
 
 org.apache.calcite
@@ -86,10 +88,6 @@
 org.apache.http
 
${shadeBase}.org.apache.http
 
-
-org.apache.commons
-
${shadeBase}.org.apache.commons
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index a1b9aef..8e69e68 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -209,7 +209,7 @@ public class KylinMeta extends MetaImpl {
 } catch (NoSuchFieldException e) {
 throw new RuntimeException(e);
 }
-columns.add(columnMetaData(name, index, field.getType()));
+columns.add(columnMetaData(name, index, field.getType(), true));
 fields.add(field);
 fieldNames.add(fieldName);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 080558b..80af108 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -43,6 +43,12 @@
 
 org.apache.kylin
 atopcalcite
+
+
+avatica-core
+org.apache.calcite.avatica
+
+
 
 
 org.apache.kylin
@@ -74,14 +80,6 @@
 org.apache.kylin
 kylin-query
 
-
-org.apache.calcite
-calcite-linq4j
-
-
-commons-cli
-commons-cli
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java 
b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index df6eb2f..05f615f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
 import java.io.File;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
+import java.sql.Driver;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.Statement;

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/pom.xml
--
diff --git a/pom.xml b/pom.xml
index ff4c4e8..57b7752 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,8 +112,8 @@
 1.8.9
 
 
-1.10.0
-1.8.0
+1.11.0
+1.9.0
 
 
 jacoco
@@ -450,15 +450,9 @@
 
 
 
-org.apache.calcite
-calcite-linq4j
-${calcite.version}
-
-
 org.apache.calcite.avatica
 avatica
 ${avatica.version}
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/query/pom.xml
--
diff --git a/query/pom.xml b/query/pom.xml
index 1dc05d1..6ab74a7 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -36,16 +36,18 @@
 
 org.apache.kylin
 atopcalcite
+
+
+org.apache.calcite.avatica
+avatica-core
+
+

[07/47] kylin git commit: KYLIN-2387 code refactor, merge ImmutableBitmapCounter and MutableBitmapCounter into RoaringBitmapCounter

[billyliu]

KYLIN-2387 code refactor, merge ImmutableBitmapCounter and MutableBitmapCounter 
into RoaringBitmapCounter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38c3e7bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38c3e7bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38c3e7bf

Branch: refs/heads/KYLIN-2361
Commit: 38c3e7bf691ecdfd0f8d42fcc97065a0596be018
Parents: 61833d9
Author: gaodayue 
Authored: Thu Jan 19 15:23:34 2017 +0800
Committer: gaodayue 
Committed: Thu Jan 19 15:43:14 2017 +0800

--
 .../gridtable/AggregationCacheMemSizeTest.java  |  11 +-
 .../metadata/measure/MeasureCodecTest.java  |   5 +-
 .../kylin/measure/bitmap/BitmapAggregator.java  |  25 +---
 .../kylin/measure/bitmap/BitmapCounter.java |  28 +++-
 .../measure/bitmap/BitmapCounterFactory.java|  30 
 .../BitmapIntersectDistinctCountAggFunc.java|  16 +-
 .../kylin/measure/bitmap/BitmapMeasureType.java |  17 ++-
 .../kylin/measure/bitmap/BitmapSerializer.java  |   9 +-
 .../measure/bitmap/ImmutableBitmapCounter.java  | 108 --
 .../measure/bitmap/MutableBitmapCounter.java|  60 
 .../measure/bitmap/RoaringBitmapCounter.java| 147 +++
 .../bitmap/RoaringBitmapCounterFactory.java |  47 ++
 .../measure/AggregatorMemEstimateTest.java  |   5 +-
 .../measure/bitmap/BitmapAggregatorTest.java|  56 ++-
 .../kylin/measure/bitmap/BitmapCounterTest.java |  32 +---
 .../measure/bitmap/BitmapSerializerTest.java|   5 +-
 16 files changed, 345 insertions(+), 256 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 63c7672..f749fb4 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -33,7 +33,9 @@ import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
 import org.apache.kylin.measure.basic.DoubleSumAggregator;
 import org.apache.kylin.measure.basic.LongSumAggregator;
 import org.apache.kylin.measure.bitmap.BitmapAggregator;
-import org.apache.kylin.measure.bitmap.MutableBitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounterFactory;
+import org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory;
 import org.apache.kylin.measure.hllc.HLLCAggregator;
 import org.apache.kylin.measure.hllc.HLLCounter;
 import org.github.jamm.MemoryMeter;
@@ -43,13 +45,14 @@ import com.google.common.base.Stopwatch;
 
 public class AggregationCacheMemSizeTest {
 private static final MemoryMeter meter = new MemoryMeter();
-private static final MutableBitmapCounter[] bitmaps = new 
MutableBitmapCounter[5];
+private static final BitmapCounterFactory bitmapFactory = 
RoaringBitmapCounterFactory.INSTANCE;
+private static final BitmapCounter[] bitmaps = new BitmapCounter[5];
 private static final Random random = new Random();
 
 // consider bitmaps with variant cardinality
 static {
 for (int i = 0; i < bitmaps.length; i++) {
-bitmaps[i] = new MutableBitmapCounter();
+bitmaps[i] = bitmapFactory.newBitmap();
 }
 
 final int totalBits = 1_000_000;
@@ -116,7 +119,7 @@ public class AggregationCacheMemSizeTest {
 }
 
 private BitmapAggregator createBitmapAggr(boolean lowCardinality) {
-MutableBitmapCounter counter = new MutableBitmapCounter();
+BitmapCounter counter = bitmapFactory.newBitmap();
 counter.orWith(lowCardinality ? bitmaps[0] : bitmaps[3]);
 
 BitmapAggregator result = new BitmapAggregator();

http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 97c9751..7129a5e 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -25,7 +25,8 @@ import java.nio.ByteBuffer;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import 

[12/47] kylin git commit: KYLIN-2394 merge CALCITE-1592

[billyliu]

KYLIN-2394 merge CALCITE-1592


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34f2d3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34f2d3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34f2d3e

Branch: refs/heads/KYLIN-2361
Commit: d34f2d3e2bf691c55077a9a76a89217ae915bbdd
Parents: dfb5fac
Author: Billy Liu 
Authored: Fri Jan 20 15:42:37 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 15:42:37 2017 +0800

--
 .../calcite/sql2rel/SqlToRelConverter.java  | 51 +---
 1 file changed, 23 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d34f2d3e/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 171e299..cc9c9a1 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -935,19 +935,21 @@ public class SqlToRelConverter {
 /**
  * Push down all the NOT logical operators into any IN/NOT IN operators.
  *
+ * @param scope Scope where {@code sqlNode} occurs
  * @param sqlNode the root node from which to look for NOT operators
  * @return the transformed SqlNode representation with NOT pushed down.
  */
-private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+private static SqlNode pushDownNotForIn(SqlValidatorScope scope,
+SqlNode sqlNode) {
 if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
 SqlCall sqlCall = (SqlCall) sqlNode;
 if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
 || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
 SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
 for (int i = 0; i < sqlOperands.length; i++) {
-sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+sqlOperands[i] = pushDownNotForIn(scope, sqlOperands[i]);
 }
-return sqlNode;
+return reg(scope, sqlNode);
 } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
 SqlNode childNode = sqlCall.operand(0);
 assert childNode instanceof SqlCall;
@@ -956,48 +958,34 @@ public class SqlToRelConverter {
 SqlNode[] andOperands = childSqlCall.getOperands();
 SqlNode[] orOperands = new SqlNode[andOperands.length];
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-andOperands[i]);
+orOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]));
 }
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] = pushDownNotForIn(orOperands[i]);
+orOperands[i] = pushDownNotForIn(scope, orOperands[i]);
 }
-return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
-orOperands[0], orOperands[1]);
+return reg(scope, 
SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], 
orOperands[1]));
 } else if (childSqlCall.getOperator() == 
SqlStdOperatorTable.OR) {
 SqlNode[] orOperands = childSqlCall.getOperands();
 SqlNode[] andOperands = new SqlNode[orOperands.length];
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-orOperands[i]);
+andOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]));
 }
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] = pushDownNotForIn(andOperands[i]);
+andOperands[i] = pushDownNotForIn(scope, 
andOperands[i]);
 }
-return 
SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
- 

[16/47] kylin git commit: KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs

[billyliu]

KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7de8aa12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7de8aa12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7de8aa12

Branch: refs/heads/KYLIN-2361
Commit: 7de8aa1203a72bad105ed692f7100535939b03af
Parents: c2229c9
Author: kangkaisen 
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: kangkaisen 
Committed: Sat Jan 21 23:19:50 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 ++-
 .../mr/steps/FactDistinctColumnsReducer.java| 240 +++
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 175 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
 }
 }
 
+public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+@Override
+public boolean accept(Path path) {
+return path.getName().startsWith(filter);
+}
+});
+
+if (fileStatus.length == 1) {
+return fileStatus[0].getPath();
+} else {
+return null;
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/statistics";
+return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
-String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
+String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+String CFG_OUTPUT_COLUMN = "column";
+String CFG_OUTPUT_DICT = "dict";
+String CFG_OUTPUT_STATISTICS = 

[08/47] kylin git commit: KYLIN-2409 change inmem cubing to single thread by default

[billyliu]

KYLIN-2409 change inmem cubing to single thread by default


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21969753
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21969753
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21969753

Branch: refs/heads/KYLIN-2361
Commit: 21969753cc66efa22bc9fc933af46346e9846631
Parents: b49c9e3
Author: shaofengshi 
Authored: Thu Jan 19 11:52:17 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:37:42 2017 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java  | 6 +-
 .../kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java| 2 +-
 .../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 9 +
 .../org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java | 2 ++
 4 files changed, 17 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d6774ff..74903d5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -300,6 +300,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
 }
 
+public int getCubeAlgorithmInMemConcurrentThreads() {
+return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-concurrent-threads", 
"1"));
+}
+
 public boolean isIgnoreCubeSignatureInconsistency() {
 return 
Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", 
"false"));
 }
@@ -744,7 +748,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getYarnStatusCheckIntervalSeconds() {
-return 
Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", 
"60"));
+return 
Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", 
"10"));
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index 651203a..c7a4a05 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -46,7 +46,7 @@ abstract public class AbstractInMemCubeBuilder {
 final protected CubeDesc cubeDesc;
 final protected Map dictionaryMap;
 
-protected int taskThreadCount = 4;
+protected int taskThreadCount = 1;
 protected int reserveMemoryMB = 100;
 
 public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc 
flatDesc, Map dictionaryMap) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 55c04c6..9158aa3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -36,6 +36,7 @@ import java.util.Map.Entry;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.common.util.Pair;
@@ -178,11 +179,16 @@ public class GTAggregateScanner implements IGTScanner {
 final List dumps;
 final int keyLength;
 final boolean[] compareMask;
+boolean compareAll = true;
 final BufferedMeasureCodec measureCodec;
 
 final Comparator bytesComparator = new Comparator() {
 @Override
 public int compare(byte[] o1, byte[] o2) {
+

[44/47] kylin git commit: KYLIN-2418 drop unused useSandbox

[billyliu]

KYLIN-2418 drop unused useSandbox


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/546f88f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/546f88f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/546f88f3

Branch: refs/heads/KYLIN-2361
Commit: 546f88f38ca017773c17ccaf2cf4812b0c7c4666
Parents: 0582512
Author: Yang Li 
Authored: Mon Jan 30 09:07:29 2017 +0800
Committer: Yang Li 
Committed: Tue Jan 31 08:11:41 2017 +0800

--
 .../kylin/common/persistence/FileResourceStore.java   |  4 
 .../org/apache/kylin/common/persistence/ResourceTool.java | 10 ++
 kylin-it/pom.xml  |  6 --
 pom.xml   |  8 
 .../apache/kylin/storage/hbase/HBaseResourceStore.java|  5 +
 5 files changed, 15 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index dcd3f38..3e012f5 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -160,4 +160,8 @@ public class FileResourceStore extends ResourceStore {
 return new File(root, resPath);
 }
 
+@Override
+public String toString() {
+return root.getAbsolutePath();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index b3aac09..6ba68ae 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -133,22 +133,24 @@ public class ResourceTool {
 ResourceStore src = ResourceStore.getStore(srcConfig);
 ResourceStore dst = ResourceStore.getStore(dstConfig);
 
+logger.info("Copy from {} to {}", src, dst);
+
 copyR(src, dst, path);
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, 
List paths) throws IOException {
 ResourceStore src = ResourceStore.getStore(srcConfig);
 ResourceStore dst = ResourceStore.getStore(dstConfig);
+
+logger.info("Copy from {} to {}", src, dst);
+
 for (String path : paths) {
 copyR(src, dst, path);
 }
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) 
throws IOException {
-
-ResourceStore src = ResourceStore.getStore(srcConfig);
-ResourceStore dst = ResourceStore.getStore(dstConfig);
-copyR(src, dst, "/");
+copy(srcConfig, dstConfig, "/");
 }
 
 public static void copyR(ResourceStore src, ResourceStore dst, String 
path) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index f88db9f..9662806 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -272,10 +272,6 @@
 
 
 
-useSandbox
-true
-
-
 log4j.configuration
 
file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
 
@@ -298,7 +294,6 @@
 test
 java
 
--DuseSandbox=true
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
 
-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
@@ -320,7 +315,6 @@
 test
 java

[22/47] kylin git commit: KYLIN 2396 GUI for Percentile pre aggregation implementation

[billyliu]

KYLIN 2396 GUI for Percentile pre aggregation implementation

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28673a81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28673a81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28673a81

Branch: refs/heads/KYLIN-2361
Commit: 28673a8126ec2ecff8c125c0ecf887ce2565362c
Parents: 6aaf667
Author: chenzhx <346839...@qq.com>
Authored: Fri Jan 20 18:00:20 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 11:32:46 2017 +0800

--
 webapp/app/js/controllers/cubeMeasures.js  | 18 +++
 webapp/app/js/model/cubeConfig.js  |  2 +-
 webapp/app/partials/cubeDesigner/measures.html | 54 -
 3 files changed, 40 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index c8548ce..7c155fe 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -80,6 +80,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   $scope.updateMeasureStatus.editIndex = index;
 }
 $scope.nextParameters = [];
+$scope.measureParamValueColumn=$scope.getCommonMetricColumns();
 $scope.newMeasure = (!!measure)? jQuery.extend(true, 
{},measure):CubeDescModel.createMeasure();
 if(!!measure && measure.function.parameter.next_parameter){
   $scope.nextPara.value = measure.function.parameter.next_parameter.value;
@@ -93,6 +94,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 }else{
   $scope.newMeasure.showDim=false;
 }
+$scope.measureParamValueUpdate();
 if($scope.newMeasure.function.expression=="TOP_N"){
   $scope.convertedColumns=[];
   if($scope.newMeasure.function.configuration==null){
@@ -326,6 +328,19 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   return false;
 }
   }
+
+  $scope.measureParamValueUpdate = function(){
+if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && 
$scope.newMeasure.showDim==true){
+   $scope.measureParamValueColumn=$scope.getAllModelDimMeasureColumns();
+}
+if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && 
$scope.newMeasure.showDim==false){
+   $scope.measureParamValueColumn=$scope.getCommonMetricColumns();
+}
+if($scope.newMeasure.function.expression == 'EXTENDED_COLUMN'){
+  $scope.measureParamValueColumn=$scope.getExtendedHostColumn();
+}
+  }
+
   //map right return type for param
   $scope.measureReturnTypeUpdate = function(){
 
@@ -393,6 +408,9 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 case "COUNT":
   $scope.newMeasure.function.returntype = "bigint";
   break;
+case "PERCENTILE":
+  $scope.newMeasure.function.returntype = "percentile(100)";
+  break;
 default:
   $scope.newMeasure.function.returntype = "";
   break;

http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/model/cubeConfig.js
--
diff --git a/webapp/app/js/model/cubeConfig.js 
b/webapp/app/js/model/cubeConfig.js
index 87413dc..35af1fc 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -20,7 +20,7 @@ KylinApp.constant('cubeConfig', {
 
   //~ Define metadata & class
   measureParamType: ['column', 'constant'],
-  measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 
'RAW','EXTENDED_COLUMN'],
+  measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 
'RAW','EXTENDED_COLUMN','PERCENTILE'],
   dimensionDataTypes: ["string", "tinyint", "int", "bigint", "date"],
   cubePartitionTypes: ['APPEND'],
   joinTypes: [

http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index 4d2d8ab..4a48843 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -104,7 +104,7 @@
   
   
   
   
@@ -144,35 +144,23 @@
 
 
   
-  

[35/47] kylin git commit: KYLIN-2415 Change back default metadata name to kylin_metadata

[billyliu]

KYLIN-2415 Change back default metadata name to kylin_metadata


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6d247c2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6d247c2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6d247c2

Branch: refs/heads/KYLIN-2361
Commit: b6d247c2c1412d7eb1024048ae7aad0ba5d1ea82
Parents: 971bf80
Author: shaofengshi 
Authored: Tue Jan 24 13:43:04 2017 +0800
Committer: shaofengshi 
Committed: Tue Jan 24 13:43:04 2017 +0800

--
 build/conf/kylin.properties| 2 +-
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
 .../apache/kylin/metadata/realization/IRealizationConstants.java   | 2 +-
 examples/test_case_data/sandbox/kylin.properties   | 2 +-
 .../org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java | 2 +-
 .../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java| 2 +-
 .../org/apache/kylin/storage/hbase/util/StorageCleanupJob.java | 2 +-
 .../main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java | 2 +-
 tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java| 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 196a711..eceb886 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -18,7 +18,7 @@
 ### METADATA | ENV ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_default_instance@hbase
+kylin.metadata.url=kylin_metadata@hbase
 
 # Working folder in HDFS, make sure user has the right access to the hdfs 
directory
 kylin.env.hdfs-working-dir=/kylin

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 75b38ff..07cb124 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -216,7 +216,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 public String getMetadataUrlPrefix() {
 String metadataUrl = getMetadataUrl();
-String defaultPrefix = "kylin_default_instance";
+String defaultPrefix = "kylin_metadata";
 
 if (metadataUrl.endsWith("@hbase")) {
 int cut = metadataUrl.lastIndexOf('@');

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index adc53a1..2a3b01e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -28,7 +28,7 @@ public class IRealizationConstants {
 
 /**
  * For each cube htable, we leverage htable's metadata to keep track of
- * which kylin server(represented by its kylin_default_instance prefix) 
owns this htable
+ * which kylin server(represented by its kylin_metadata prefix) owns this 
htable
  */
 public final static String HTableTag = "KYLIN_HOST";
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 2c2da91..fa0c6bc 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -39,7 +39,7 @@ kylin.source.hive.client=cli
 ### STORAGE ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_default_instance@hbase
+kylin.metadata.url=kylin_metadata@hbase
 
 
 # The storage for final cube file in hbase

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 

[43/47] kylin git commit: minor, make DefaultSchedulerTest.testDiscard() more stable

[billyliu]

minor, make DefaultSchedulerTest.testDiscard() more stable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0582512e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0582512e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0582512e

Branch: refs/heads/KYLIN-2361
Commit: 0582512ea06bcd1c662903ff69012ba1cc7cc33c
Parents: c3fff6d
Author: Li Yang 
Authored: Thu Jan 26 11:49:04 2017 +0800
Committer: Li Yang 
Committed: Thu Jan 26 11:49:56 2017 +0800

--
 .../apache/kylin/job/SelfStopExecutable.java| 31 
 .../impl/threadpool/DefaultSchedulerTest.java   |  5 ++--
 2 files changed, 27 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java 
b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
index b4f6a98..9a3eb48 100644
--- a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
+++ b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
@@ -26,21 +26,40 @@ import org.apache.kylin.job.execution.ExecuteResult;
  */
 public class SelfStopExecutable extends BaseTestExecutable {
 
+volatile boolean doingWork;
+
 public SelfStopExecutable() {
 super();
 }
 
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
+doingWork = true;
 try {
-Thread.sleep(5000);
-} catch (InterruptedException e) {
-}
-if (isDiscarded()) {
-return new ExecuteResult(ExecuteResult.State.STOPPED, "stopped");
-} else {
+for (int i = 0; i < 20; i++) {
+sleepOneSecond();
+
+if (isDiscarded())
+return new ExecuteResult(ExecuteResult.State.STOPPED, 
"stopped");
+}
+
 return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+} finally {
+doingWork = false;
+}
+}
+
+private void sleepOneSecond() {
+try {
+Thread.sleep(1000);
+} catch (InterruptedException e) {
+throw new RuntimeException(e);
 }
 }
 
+public void waitForDoWork() {
+while (doingWork) {
+sleepOneSecond();
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index bcd6a59..2416311 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -97,7 +97,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest {
 @Test
 public void testDiscard() throws Exception {
 DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SelfStopExecutable();
+SelfStopExecutable task1 = new SelfStopExecutable();
 job.addTask(task1);
 jobService.addJob(job);
 waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
@@ -105,8 +105,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest 
{
 waitForJobFinish(job.getId());
 Assert.assertEquals(ExecutableState.DISCARDED, 
jobService.getOutput(job.getId()).getState());
 Assert.assertEquals(ExecutableState.DISCARDED, 
jobService.getOutput(task1.getId()).getState());
-Thread.sleep(5000);
-System.out.println(job);
+task1.waitForDoWork();
 }
 
 @SuppressWarnings("rawtypes")

[18/47] kylin git commit: KYLIN-2412 Unclosed DataOutputStream in RoaringBitmapCounter#write()

[billyliu]

KYLIN-2412 Unclosed DataOutputStream in RoaringBitmapCounter#write()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d264339b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d264339b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d264339b

Branch: refs/heads/KYLIN-2361
Commit: d264339b1c16c195ffafc2217b793d81bdbd6434
Parents: 7576a09
Author: gaodayue 
Authored: Sun Jan 22 14:02:09 2017 +0800
Committer: gaodayue 
Committed: Sun Jan 22 14:02:09 2017 +0800

--
 .../org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java| 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d264339b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index cd07d20..fb9dcfc 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -107,7 +107,9 @@ public class RoaringBitmapCounter implements BitmapCounter {
 if (out.remaining() < bitmap.serializedSizeInBytes()) {
 throw new BufferOverflowException();
 }
-bitmap.serialize(new DataOutputStream(new 
ByteBufferOutputStream(out)));
+try (DataOutputStream dos = new DataOutputStream(new 
ByteBufferOutputStream(out))) {
+bitmap.serialize(dos);
+}
 }
 
 @Override

[28/47] kylin git commit: minor, refine cdh profile name

[billyliu]

minor, refine cdh profile name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db85d667
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db85d667
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db85d667

Branch: refs/heads/KYLIN-2361
Commit: db85d6679c5e1c067cfe2f1c58de0efb88044203
Parents: 87d1039
Author: Hongbin Ma 
Authored: Mon Jan 23 16:14:33 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 16:14:33 2017 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db85d667/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 520ac0c..bd1882e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1178,7 +1178,7 @@
 
 
 
-cdh-5.7
+cdh5.7
 
 2.6.0-cdh5.7.0
 2.6.0-cdh5.7.0

[34/47] kylin git commit: Add HDFSResourceStore to ResourceStore

[billyliu]

Add HDFSResourceStore to ResourceStore

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c33e38f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c33e38f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c33e38f

Branch: refs/heads/KYLIN-2361
Commit: 4c33e38f3e6c33438ef3bb830875b8ed74ffac5b
Parents: d3276e2
Author: xiefan46 <958034...@qq.com>
Authored: Mon Jan 23 10:51:12 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../kylin/common/persistence/ResourceStore.java   |  5 +
 .../kylin/storage/hdfs/ITHDFSResourceStoreTest.java   | 13 -
 .../apache/kylin/storage/hdfs/ITLockManagerTest.java  | 14 +-
 .../kylin/storage/hbase/HBaseResourceStore.java   |  3 ++-
 .../apache/kylin/storage/hdfs/HDFSResourceStore.java  |  4 +++-
 5 files changed, 23 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 9549569..25a0801 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -75,6 +75,11 @@ abstract public class ResourceStore {
 } catch (Throwable e) {
 logger.warn("Failed to load HBaseResourceStore impl class: " + 
e.toString());
 }
+try {
+
knownImpl.add(ClassUtil.forName("org.apache.kylin.storage.hdfs.HDFSResourceStore",
 ResourceStore.class));
+} catch (Throwable e) {
+logger.warn("Failed to load HDFSResourceStore impl class: " + 
e.toString());
+}
 }
 return knownImpl;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 27d8a3c..47afb01 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -19,17 +19,12 @@
 package org.apache.kylin.storage.hdfs;
 
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.ResourceStoreTest;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-
-/**
- * Created by xiefan on 17-1-10.
- */
 public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
 
 KylinConfig kylinConfig;
@@ -45,13 +40,13 @@ public class ITHDFSResourceStoreTest extends 
HBaseMetadataTestCase {
 this.cleanupTestMetadata();
 }
 
-
 @Test
 public void testResourceStoreBasic() throws Exception {
-KylinConfig config = KylinConfig.getInstanceFromEnv();
-ResourceStore store = new HDFSResourceStore(config);
+String oldUrl = kylinConfig.getMetadataUrl();
+kylinConfig.setProperty("kylin.metadata.url", 
"kylin_default_instance@hdfs");
+HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
 ResourceStoreTest.testAStore(store);
+kylinConfig.setProperty("kylin.metadata.url", oldUrl);
 }
 
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
index 2b58d30..a6cf43b 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
@@ -27,6 +27,8 @@ import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -58,6 +60,8 @@ public class ITLockManagerTest extends 

[47/47] kylin git commit: KYLIN-2361 add Tomcat8 ordered class loader

[billyliu]

KYLIN-2361 add Tomcat8 ordered class loader


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a3bd71c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a3bd71c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a3bd71c

Branch: refs/heads/KYLIN-2361
Commit: 9a3bd71c8e5ce9dc13e38560efc556dc862819a1
Parents: 2b60ac6
Author: Billy Liu 
Authored: Sat Feb 4 11:41:49 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 11:42:02 2017 +0800

--
 .../kylin/ext/CustomizedWebappClassloader.java  |   4 +-
 .../kylin/ext/OrderedWebResourceRoot.java   | 286 +++
 .../kylin/ext/WebappOrderedClassLoader.java |  66 +
 3 files changed, 353 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
--
diff --git 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
index f241865..bbf4053 100644
--- 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
+++ 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
@@ -18,14 +18,12 @@
 
 package org.apache.kylin.ext;
 
-import org.apache.catalina.loader.ParallelWebappClassLoader;
-
 /**
  * simple extension to standard ParallelWebappClassLoader
  * the only difference is that CustomizedWebappClassloader is able to delegate 
more packages
  * to parent classloaders
  */
-public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
+public class CustomizedWebappClassloader extends WebappOrderedClassLoader {
 /**
  * Set of package names which are not allowed to be loaded from a webapp
  * class loader without delegating first.

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
--
diff --git 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
new file mode 100644
index 000..9784bd8
--- /dev/null
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.ext;
+
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.catalina.Context;
+import org.apache.catalina.LifecycleException;
+import org.apache.catalina.LifecycleListener;
+import org.apache.catalina.LifecycleState;
+import org.apache.catalina.TrackedWebResource;
+import org.apache.catalina.WebResource;
+import org.apache.catalina.WebResourceRoot;
+import org.apache.catalina.WebResourceSet;
+
+public class OrderedWebResourceRoot implements WebResourceRoot {
+
+private static final String WEB_INF_LIB_PATH = "/WEB-INF/lib";
+
+private static final Comparator WEB_RESOURCE_COMPARATOR = new 
Comparator() {
+@Override
+public int compare(WebResource o1, WebResource o2) {
+return o1.getName().compareTo(o2.getName());
+}
+};
+
+private WebResourceRoot delegate;
+
+public OrderedWebResourceRoot(WebResourceRoot delegate) {
+this.delegate = delegate;
+}
+
+@Override
+public WebResource[] listResources(String path) {
+WebResource[] webResources = delegate.listResources(path);
+
+if (WEB_INF_LIB_PATH.equals(path)) {
+Arrays.sort(webResources, WEB_RESOURCE_COMPARATOR);
+}
+
+return webResources;
+}
+
+@Override
+public void addLifecycleListener(LifecycleListener listener) {
+

[31/47] kylin git commit: KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs

[billyliu]

KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6dda4fe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6dda4fe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6dda4fe

Branch: refs/heads/KYLIN-2361
Commit: f6dda4fecd5a882a17379987445a5aa0a347ff63
Parents: 4c33e38
Author: kangkaisen 
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 ++-
 .../mr/steps/FactDistinctColumnsReducer.java| 240 +++
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 175 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
 }
 }
 
+public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+@Override
+public boolean accept(Path path) {
+return path.getName().startsWith(filter);
+}
+});
+
+if (fileStatus.length == 1) {
+return fileStatus[0].getPath();
+} else {
+return null;
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/statistics";
+return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
-String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
+String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+String CFG_OUTPUT_COLUMN = "column";
+String CFG_OUTPUT_DICT = "dict";
+String 

[33/47] kylin git commit: Add Zookeeper Lock

[billyliu]

Add Zookeeper Lock

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d3276e2e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d3276e2e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d3276e2e

Branch: refs/heads/KYLIN-2361
Commit: d3276e2e909d3001724ee8fda1304ae8b7f08c63
Parents: d23bf93
Author: xiefan46 <958034...@qq.com>
Authored: Fri Jan 20 09:48:17 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  22 +-
 .../test_case_data/sandbox/kylin.properties |   4 +
 .../storage/hdfs/ITHDFSResourceStoreTest.java   |  66 +-
 .../kylin/storage/hdfs/ITLockManagerTest.java   | 205 +++
 .../kylin/storage/hbase/HBaseResourceStore.java |   1 +
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 
 .../kylin/storage/hdfs/HDFSResourceStore.java   |  95 +++--
 .../apache/kylin/storage/hdfs/LockManager.java  | 116 +++
 .../apache/kylin/storage/hdfs/ResourceLock.java |  51 +
 10 files changed, 471 insertions(+), 175 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 44d636d..75b38ff 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,6 +193,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
+public String getRawHdfsWorkingDirectory() {
+String root = getRequired("kylin.env.hdfs-working-dir");
+if (!root.endsWith("/")) {
+root += "/";
+}
+return root;
+}
+
 // 

 // METADATA
 // 

@@ -201,11 +209,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metadata.url");
 }
 
-//for hdfs resource store
-public String getHDFSMetadataUrl() {
-return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
-}
-
 // for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);
@@ -925,4 +928,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
 }
 
+//zoo keeper
+public String getZooKeeperHost() {
+return getOptional("kylin.storage-zookeeper.host", "localhost");
+}
+
+public String getZooKeeperPort() {
+return getOptional("kylin.storage-zookeeper.port", "2181");
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index b01c377..2c2da91 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -177,3 +177,7 @@ 
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
 #kylin.engine.spark-conf.spark.yarn.queue=default
 
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
+
+
+#zoo keeper
+kylin.storage-zookeeper.host=sandbox
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index ef04957..27d8a3c 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ 

[46/47] kylin git commit: KYLIN-2361 Upgrade Tomcat 8.5.9

[billyliu]

KYLIN-2361 Upgrade Tomcat 8.5.9


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b60ac6a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b60ac6a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b60ac6a

Branch: refs/heads/KYLIN-2361
Commit: 2b60ac6a42741ca70d63e6680a0fbe9aeed7d46e
Parents: a058bfb
Author: Billy Liu 
Authored: Wed Jan 18 17:19:04 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 11:42:02 2017 +0800

--
 build/script/download-tomcat.sh |  8 
 pom.xml |  2 +-
 .../java/org/apache/kylin/rest/DebugTomcat.java | 16 +---
 .../kylin/ext/CustomizedWebappClassloader.java  |  5 ++---
 4 files changed, 16 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/build/script/download-tomcat.sh
--
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index b3aa509..bdfe351 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -27,19 +27,19 @@ if [[ `uname -a` =~ "Darwin" ]]; then
 alias md5cmd="md5 -q"
 fi
 
-tomcat_pkg_version="7.0.69"
-tomcat_pkg_md5="10a071e5169a1a8b14ff35a0ad181052"
+tomcat_pkg_version="8.5.9"
+tomcat_pkg_md5="b41270a64b7774c964e4bec813eea2ed"
 
 if [ ! -f "build/apache-tomcat-${tomcat_pkg_version}.tar.gz" ]
 then
 echo "no binary file found"
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "Download tomcat failed"
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "Download tomcat failed"
 else
 if [ `md5cmd build/apache-tomcat-${tomcat_pkg_version}.tar.gz | awk 
'{print $1}'` != "${tomcat_pkg_md5}" ]
 then
 echo "md5 check failed"
 rm build/apache-tomcat-${tomcat_pkg_version}.tar.gz
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "download tomcat failed"
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "download tomcat failed"
 fi
 fi
 unalias md5cmd

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index bf33e07..b82eee2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@
 3.2.4
 2.4.0
 2.5
-7.0.69
+8.5.9
 3.1
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
--
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java 
b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 3461e1d..1b47f79 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -21,11 +21,13 @@ package org.apache.kylin.rest;
 import org.apache.catalina.Context;
 import org.apache.catalina.core.AprLifecycleListener;
 import org.apache.catalina.core.StandardServer;
-import org.apache.catalina.deploy.ErrorPage;
 import org.apache.catalina.startup.Tomcat;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.tomcat.JarScanFilter;
+import org.apache.tomcat.JarScanType;
+import org.apache.tomcat.util.descriptor.web.ErrorPage;
 
 import java.io.File;
 import java.lang.reflect.Field;
@@ -127,12 +129,12 @@ public class DebugTomcat {
 notFound.setLocation("/index.html");
 webContext.addErrorPage(notFound);
 webContext.addWelcomeFile("index.html");
-//webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
-//@Override
-//public boolean check(JarScanType arg0, String arg1) {
-//return false;
-//}
-//});
+webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
+@Override
+public boolean check(JarScanType arg0, String arg1) {
+return false;
+}
+});
 
 // tomcat start
 tomcat.start();

[29/47] kylin git commit: KYLIN-2217 use column’s identity as path name and add more logging info

[billyliu]

KYLIN-2217 use column’s identity as path name and add more logging info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e217ae84
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e217ae84
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e217ae84

Branch: refs/heads/KYLIN-2361
Commit: e217ae84e1bd163658176093de208fcdf3bdf43c
Parents: f6dda4f
Author: shaofengshi 
Authored: Sun Jan 22 09:43:10 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../java/org/apache/kylin/common/util/HadoopUtil.java|  4 
 .../apache/kylin/cube/cli/DictionaryGeneratorCLI.java|  3 +++
 .../org/apache/kylin/engine/mr/SortedColumnDFSFile.java  |  4 
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java |  2 +-
 .../kylin/engine/mr/steps/CreateDictionaryJob.java   |  9 +++--
 .../engine/mr/steps/FactDistinctColumnsReducer.java  | 11 +++
 .../engine/mr/steps/UpdateCubeInfoAfterBuildStep.java|  4 ++--
 .../hive/cardinality/HiveColumnCardinalityJob.java   |  3 +++
 8 files changed, 31 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..f242515 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -143,6 +143,10 @@ public class HadoopUtil {
 }
 
 public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+if (fs.exists(baseDir) == false) {
+return null;
+}
+
 FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
 @Override
 public boolean accept(Path path) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 3e1ab0d..2e5a38e 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -63,11 +63,14 @@ public class DictionaryGeneratorCLI {
 if (dictProvider != null) {
 Dictionary dict = dictProvider.getDictionary(col);
 if (dict != null) {
+logger.debug("Dict for '" + col.getName() + "' has already 
been built, save it");
 cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);
 } else {
+logger.debug("Dict for '" + col.getName() + "' not 
pre-built, build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 } else {
+logger.debug("Dict for '" + col.getName() + "' not pre-built, 
build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
index d3f5cdc..f396b5a 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
@@ -128,4 +128,8 @@ public class SortedColumnDFSFile implements ReadableTable {
 return comparator;
 }
 
+@Override
+public String toString() {
+return dfsPath;
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 567c1d0..44686d6 100644
--- 

[23/47] kylin git commit: add back calcite linq4j

[billyliu]

add back calcite linq4j


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68e87f08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68e87f08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68e87f08

Branch: refs/heads/KYLIN-2361
Commit: 68e87f08255caf33dd7946f43b3393372a73c68b
Parents: b56bc62
Author: Hongbin Ma 
Authored: Mon Jan 23 11:54:54 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 11:54:54 2017 +0800

--
 pom.xml | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/68e87f08/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 57b7752..e590fd7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -450,6 +450,11 @@
 
 
 
+org.apache.calcite
+calcite-linq4j
+${calcite.version}
+
+
 org.apache.calcite.avatica
 avatica
 ${avatica.version}

[39/47] kylin git commit: KYLIN-2374 minor, rename zookeeper config properties

[billyliu]

KYLIN-2374 minor,rename zookeeper config properties

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2a59c43
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2a59c43
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2a59c43

Branch: refs/heads/KYLIN-2361
Commit: a2a59c4303a29de308beb7b5c8300f78023313dd
Parents: 5629341
Author: xiefan46 <958034...@qq.com>
Authored: Tue Jan 24 18:57:07 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Jan 25 14:21:37 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java  | 4 ++--
 examples/test_case_data/sandbox/kylin.properties| 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a2a59c43/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 07cb124..5932197 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -930,11 +930,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 //zoo keeper
 public String getZooKeeperHost() {
-return getOptional("kylin.storage-zookeeper.host", "localhost");
+return getOptional("kylin.storage.lock-manager-zk-host", "localhost");
 }
 
 public String getZooKeeperPort() {
-return getOptional("kylin.storage-zookeeper.port", "2181");
+return getOptional("kylin.storage.lock-manager-zk-port", "2181");
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a2a59c43/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index fa0c6bc..6cb5148 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -55,6 +55,9 @@ kylin.env.hdfs-working-dir=/kylin
 
 kylin.engine.mr.reduce-input-mb=500
 
+#zoo keeper host in lock manager
+kylin.storage.lock-manager-zk-host=sandbox
+
 ### JOB ###
 
 # max job retry on error, default 0: no retry
@@ -179,5 +182,3 @@ 
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
 
 
-#zoo keeper
-kylin.storage-zookeeper.host=sandbox
\ No newline at end of file

[41/47] kylin git commit: KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217

[billyliu]

KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6a9382b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6a9382b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6a9382b

Branch: refs/heads/KYLIN-2361
Commit: e6a9382b8d357e5bb13be2f678ebabb8bad75dc7
Parents: 5eae37e
Author: xiefan46 <958034...@qq.com>
Authored: Tue Jan 24 11:14:40 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 09:56:54 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 3 ---
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java  | 6 ++
 2 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b1acbbf..b25bcc0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -741,9 +741,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public boolean isReducerLocalBuildDict() {
-if (getUHCReducerCount() != 1) {
-return false;
-}
 return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8c56bdf..cf94b30 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -132,6 +132,12 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
+int[] uhcIndex = 
CubeManager.getInstance(config).getUHCIndex(cubeDesc);
+int colIndex = reducerIdToColumnIndex.get(taskId);
+if (uhcIndex[colIndex] == 1)
+isReducerLocalBuildDict = false; //for UHC columns, this 
feature should be disabled
+}
 if (isReducerLocalBuildDict) {
 builder = 
DictionaryGenerator.newDictionaryBuilder(col.getType());
 builder.init(null, 0);

[27/47] kylin git commit: KYLIN-2413 update test conf files

[billyliu]

KYLIN-2413 update test conf files


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87d10394
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87d10394
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87d10394

Branch: refs/heads/KYLIN-2361
Commit: 87d103948fdf8951941c9a7fd11213dd569f8aa5
Parents: eb73da7
Author: lidongsjtu 
Authored: Mon Jan 23 13:12:36 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 13:18:24 2017 +0800

--
 dev-support/sync_hbase_cdh_branches.sh  |  15 +--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |  25 
 dev-support/test_all_against_hdp_2_4_0_0_169.sh |  25 
 .../sandbox/capacity-scheduler.xml  |  17 +--
 examples/test_case_data/sandbox/core-site.xml   |  35 -
 .../test_case_data/sandbox/hadoop-policy.xml|   2 +-
 .../test_case_data/sandbox/hbase-policy.xml |   2 +-
 examples/test_case_data/sandbox/hbase-site.xml  | 100 +--
 examples/test_case_data/sandbox/hdfs-site.xml   |  84 +++-
 examples/test_case_data/sandbox/hive-site.xml   |  89 -
 examples/test_case_data/sandbox/mapred-site.xml |  57 ++---
 examples/test_case_data/sandbox/yarn-site.xml   | 127 ---
 12 files changed, 404 insertions(+), 174 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh 
b/dev-support/sync_hbase_cdh_branches.sh
index 472f402..592053a 100755
--- a/dev-support/sync_hbase_cdh_branches.sh
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -37,24 +37,17 @@ trap 'error ${LINENO} ${?}' ERR
 # 
 
 git fetch apache
-git checkout apache/$base-hbase1.x
-git format-patch -1
-git checkout apache/$base-cdh5.7
+git checkout apache/$base-hbase0.98
 git format-patch -1
 
 git checkout apache/$base
 git checkout -b tmp
 git reset apache/$base --hard
 
-git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-mvn clean compile -DskipTests
-git push apache tmp:$base-hbase1.x -f
-rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-
-git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+git am -3 --ignore-whitespace 
0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
 mvn clean compile -DskipTests
-git push apache tmp:$base-cdh5.7 -f
-rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+git push apache tmp:$base-hbase0.98 -f
+rm 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
 
 # clean up
 git checkout master

http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
deleted file mode 100755
index f7780dd..000
--- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-cd ${dir}
-cd ..
-
-mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee 
mvnverify.log

http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_4_0_0_169.sh
--
diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh 
b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
new file mode 100755
index 000..2a3d24b
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding 

[03/47] kylin git commit: KYLIN-2410 remove force qualified path in HadoopUtil.getFileSystem

[billyliu]

KYLIN-2410 remove force qualified path in HadoopUtil.getFileSystem


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61833d95
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61833d95
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61833d95

Branch: refs/heads/KYLIN-2361
Commit: 61833d952daa3ed7e3938f6e310caca5a52a4974
Parents: eddb695
Author: Yang Li 
Authored: Thu Jan 19 10:35:50 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 19 12:42:44 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/util/HadoopUtil.java| 3 ---
 1 file changed, 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61833d95/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 390c209..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -82,9 +82,6 @@ public class HadoopUtil {
 }
 
 public static FileSystem getFileSystem(Path path, Configuration conf) {
-if (StringUtils.isBlank(path.toUri().getScheme()))
-throw new IllegalArgumentException("Path must be qualified: " + 
path);
-
 try {
 return path.getFileSystem(conf);
 } catch (IOException e) {

[38/47] kylin git commit: KYLIN-2374 fix path bug in HDFSResourceStore

[billyliu]

KYLIN-2374 fix path bug in HDFSResourceStore

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56293411
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56293411
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56293411

Branch: refs/heads/KYLIN-2361
Commit: 5629341122abe18d9576f71a0ea99ac83c327c21
Parents: a86569a
Author: xiefan46 <958034...@qq.com>
Authored: Wed Jan 25 10:41:40 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Jan 25 11:19:13 2017 +0800

--
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 23 +++-
 1 file changed, 13 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56293411/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index 0ad6964..38acfb0 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -18,7 +18,15 @@
 
 package org.apache.kylin.storage.hdfs;
 
-import com.google.common.collect.Lists;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -32,14 +40,7 @@ import org.apache.kylin.common.util.HadoopUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Lists;
 
 public class HDFSResourceStore extends ResourceStore {
 
@@ -63,7 +64,7 @@ public class HDFSResourceStore extends ResourceStore {
 int cut = metadataUrl.indexOf('@');
 String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : 
metadataUrl.substring(0, cut);
 String hdfsUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 
1);
-if(!hdfsUrl.equals("hdfs"))
+if (!hdfsUrl.equals("hdfs"))
 throw new IOException("Can not create HDFSResourceStore. Url not 
match. Url:" + hdfsUrl);
 metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME;
 logger.info("meta dir name :" + metaDirName);
@@ -245,6 +246,8 @@ public class HDFSResourceStore extends ResourceStore {
 }
 
 private Path getRealHDFSPath(String resourcePath) {
+if (resourcePath.equals("/"))
+return this.hdfsMetaPath;
 if (resourcePath.startsWith("/") && resourcePath.length() > 1)
 resourcePath = resourcePath.substring(1, resourcePath.length());
 return new Path(this.hdfsMetaPath, resourcePath);

[17/47] kylin git commit: Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs"

[billyliu]

Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs"

This reverts commit 7de8aa1203a72bad105ed692f7100535939b03af.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7576a09f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7576a09f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7576a09f

Branch: refs/heads/KYLIN-2361
Commit: 7576a09f29b75f2eb80a1b5ca9c65fe5c81b1144
Parents: 7de8aa1
Author: shaofengshi 
Authored: Sun Jan 22 11:38:17 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 11:38:17 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 --
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 +--
 .../mr/steps/FactDistinctColumnsReducer.java| 240 ---
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 187 insertions(+), 175 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,10 +26,8 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -142,18 +140,4 @@ public class HadoopUtil {
 }
 }
 
-public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
-FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
-@Override
-public boolean accept(Path path) {
-return path.getName().startsWith(filter);
-}
-});
-
-if (fileStatus.length == 1) {
-return fileStatus[0].getPath();
-} else {
-return null;
-}
-}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index c34a904..696b22a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
+return getRealizationRootPath(jobId) + "/statistics";
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 602b4bb..0281539 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,16 +53,9 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
+String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
-String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
-
-String CFG_OUTPUT_COLUMN = 

[13/47] kylin git commit: KYLIN 2370 Refine unload and reload table

[billyliu]

KYLIN 2370 Refine unload and reload table

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a853a7c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a853a7c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a853a7c4

Branch: refs/heads/KYLIN-2361
Commit: a853a7c4b6b7d42cc3ffd066b618f5ea72ca0d9f
Parents: 85a1eb3
Author: chenzhx <346839...@qq.com>
Authored: Mon Jan 9 15:09:16 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 16:21:11 2017 +0800

--
 webapp/app/css/AdminLTE.css |  27 +++
 webapp/app/js/controllers/sourceMeta.js | 175 ---
 .../js/directives/kylin_abn_tree_directive.js   |   7 +-
 .../app/partials/tables/source_table_tree.html  |   6 +-
 webapp/app/partials/tables/table_unload.html|  33 
 5 files changed, 111 insertions(+), 137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/css/AdminLTE.css
--
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index 857dbf7..c7740d1 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -4800,3 +4800,30 @@ Gradient Background colors
 white-space: normal !important;
   }
 }
+
+.abn-tree .abn-tree-row .tree-table-btn{
+width: 4%;
+float: right;
+border-radius: 5px;
+margin-right: 5px;
+padding: 3px 16px 2px 5px;
+color: #ff;
+position:static;
+}
+.abn-tree .abn-tree-row .tree-table-btn .tooltip{
+overflow:visible;
+}
+.abn-tree .abn-tree-row .btn-info:hover{
+   background-color:#269abc;
+}
+.abn-tree > .abn-tree-row.active > .btn-info{
+background-color:#269abc;
+border-left-color:#269abc;
+}
+.abn-tree .abn-tree-row .btn-success:hover{
+background-color:#008d4c;
+}
+.abn-tree > .abn-tree-row.active > .btn-success{
+background-color:#008d4c;
+border-left-color:#008d4c;
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/js/controllers/sourceMeta.js
--
diff --git a/webapp/app/js/controllers/sourceMeta.js 
b/webapp/app/js/controllers/sourceMeta.js
index a53a35f..5421673 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -132,31 +132,83 @@ KylinApp
   });
 };
 
-$scope.openUnLoadModal = function () {
-  if(!$scope.projectModel.selectedProject){
-SweetAlert.swal('Oops...', "Please select a project.", 'info');
+$scope.reloadTable = function (tableName,projectName){
+  loadingRequest.show();
+  TableService.loadHiveTable({tableName: tableName, action: projectName}, 
{calculate: $scope.isCalculate}, function (result) {
+var loadTableInfo = "";
+angular.forEach(result['result.loaded'], function (table) {
+  loadTableInfo += "\n" + table;
+})
+var unloadedTableInfo = "";
+angular.forEach(result['result.unloaded'], function (table) {
+  unloadedTableInfo += "\n" + table;
+})
+if (result['result.unloaded'].length != 0 && 
result['result.loaded'].length == 0) {
+  SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + 
unloadedTableInfo, 'error');
+}
+if (result['result.loaded'].length != 0 && 
result['result.unloaded'].length == 0) {
+  SweetAlert.swal('Success!', 'The following table(s) have been 
successfully loaded: ' + loadTableInfo, 'success');
+}
+if (result['result.loaded'].length != 0 && 
result['result.unloaded'].length != 0) {
+  SweetAlert.swal('Partial loaded!', 'The following table(s) have been 
successfully loaded: ' + loadTableInfo + "\n\n Failed to load following 
table(s):" + unloadedTableInfo, 'warning');
+}
+loadingRequest.hide();
+$scope.aceSrcTbLoaded(true);
+  }, function (e) {
+if (e.data && e.data.exception) {
+  var message = e.data.exception;
+  var msg = !!(message) ? message : 'Failed to take action.';
+  SweetAlert.swal('Oops...', msg, 'error');
+} else {
+  SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+}
+loadingRequest.hide();
+  })
+}
+
+
+
+$scope.removeList = function (tableName,projectName) {
+  if (tableName.trim() === "") {
+SweetAlert.swal('', 'Please input table(s) you want to unload.', 
'info');
 return;
   }
-  $modal.open({
-templateUrl: 'removeHiveTable.html',
-controller: ModalInstanceCtrl,
-backdrop : 'static',
-resolve: {
-  tableNames: function () {
-return $scope.tableNames;

[19/47] kylin git commit: Merge branch 'KYLIN-2394'

[billyliu]

Merge branch 'KYLIN-2394'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c4591a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c4591a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c4591a6

Branch: refs/heads/KYLIN-2361
Commit: 2c4591a609c8d8141c880722a3b8b282b785133e
Parents: d264339 d34f2d3
Author: Billy Liu 
Authored: Sun Jan 22 17:43:16 2017 +0800
Committer: Billy Liu 
Committed: Sun Jan 22 17:43:16 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3782 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2348 insertions(+), 1494 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2c4591a6/kylin-it/pom.xml
--

[20/47] kylin git commit: minor, add config kylin.cube.allow-appear-in-multiple-projects

[billyliu]

minor, add config kylin.cube.allow-appear-in-multiple-projects


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6aaf6676
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6aaf6676
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6aaf6676

Branch: refs/heads/KYLIN-2361
Commit: 6aaf6676b5e904260f16d1222bfe99013a095963
Parents: 2c4591a
Author: Hongbin Ma 
Authored: Mon Jan 23 11:31:38 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 11:31:38 2017 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java  | 9 ++---
 .../src/main/java/org/apache/kylin/engine/mr/CubingJob.java | 7 ++-
 2 files changed, 12 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6aaf6676/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 74903d5..05df177 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -181,7 +181,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 if (!root.endsWith("/")) {
 root += "/";
 }
-
+
 // make sure path qualified
 if (!root.contains("://")) {
 if (!root.startsWith("/"))
@@ -189,7 +189,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 else
 root = "hdfs://" + root;
 }
-
+
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
@@ -320,6 +320,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
 }
 
+public boolean allowCubeAppearInMultipleProjects() {
+return 
Boolean.parseBoolean(getOptional("kylin.cube.allow-appear-in-multiple-projects",
 "false"));
+}
+
 // 

 // JOB
 // 

@@ -767,7 +771,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "10.0"));
 }
 
-
 public int getSparkMinPartition() {
 return Integer.valueOf(getOptional("kylin.engine.spark.min-partition", 
"1"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6aaf6676/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 64e9edb..5aa7d72 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -84,7 +84,12 @@ public class CubingJob extends DefaultChainedExecutable {
 if (projList == null || projList.size() == 0) {
 throw new RuntimeException("Cannot find the project containing the 
cube " + cube.getName() + "!!!");
 } else if (projList.size() >= 2) {
-throw new RuntimeException("Find more than one project containing 
the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! 
");
+String msg = "Find more than one project containing the cube " + 
cube.getName() + ". It does't meet the uniqueness requirement!!! ";
+if (!config.getConfig().allowCubeAppearInMultipleProjects()) {
+throw new RuntimeException(msg);
+} else {
+logger.warn(msg);
+}
 }
 
 CubingJob result = new CubingJob();

[26/47] kylin git commit: KYLIN-2413 a new profile for cdh binary packages

[billyliu]

KYLIN-2413 a new profile for cdh binary packages


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb73da7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb73da7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb73da7a

Branch: refs/heads/KYLIN-2361
Commit: eb73da7a0b27540af54fec23b561b390405620c5
Parents: 169c3de
Author: lidongsjtu 
Authored: Mon Jan 23 11:30:13 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 13:03:29 2017 +0800

--
 build/script/build.sh   |   2 +-
 build/script/package.sh |  11 +-
 .../kylin/engine/mr/steps/MockupMapContext.java |  11 +-
 pom.xml | 114 +++
 .../storage/hbase/steps/MockupMapContext.java   |  17 +--
 tool/pom.xml|  10 ++
 6 files changed, 145 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/build/script/build.sh
--
diff --git a/build/script/build.sh b/build/script/build.sh
index c68ca94..458c612 100755
--- a/build/script/build.sh
+++ b/build/script/build.sh
@@ -22,7 +22,7 @@ echo "package back-end"
 dir=$(dirname ${0})
 cd ${dir}/../..
 
-mvn clean install -DskipTests   || { exit 1; }
+mvn clean install -DskipTests $@ || { exit 1; }
 
 #package webapp
 echo 'package front-end'

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/build/script/package.sh
--
diff --git a/build/script/package.sh b/build/script/package.sh
index c850ec3..a58a5ce 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -74,9 +74,8 @@ cat << EOF > build/commit_SHA1
 EOF
 git rev-parse HEAD >> build/commit_SHA1
 
-sh build/script/build.sh || { exit 1; }
-sh build/script/download-tomcat.sh || { exit 1; }
-sh build/script/download-spark.sh || { exit 1; }
-sh build/script/prepare.sh || { exit 1; }
-sh build/script/compress.sh || { exit 1; }
-
+sh build/script/build.sh $@ || { exit 1; }
+sh build/script/download-tomcat.sh  || { exit 1; }
+sh build/script/download-spark.sh   || { exit 1; }
+sh build/script/prepare.sh  || { exit 1; }
+sh build/script/compress.sh || { exit 1; }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index dfbba14..92115c9 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -47,7 +47,7 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
  * 
  */
 @SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext implements MapContext{
+public class MockupMapContext implements MapContext {
 
 private Configuration hconf;
 
@@ -60,7 +60,7 @@ public class MockupMapContext implements MapContext{
 return new WrappedMapper().getMapContext(new MockupMapContext(hconf, 
outKV));
 }
 
-public MockupMapContext(Configuration hconf, Object[] outKV){
+public MockupMapContext(Configuration hconf, Object[] outKV) {
 this.hconf = hconf;
 this.outKV = outKV;
 }
@@ -109,7 +109,6 @@ public class MockupMapContext implements MapContext{
 throw new NotImplementedException();
 }
 
-@Override
 public float getProgress() {
 throw new NotImplementedException();
 }
@@ -224,7 +223,6 @@ public class MockupMapContext implements MapContext{
 throw new NotImplementedException();
 }
 
-@Override
 public boolean getTaskCleanupNeeded() {
 throw new NotImplementedException();
 }
@@ -239,7 +237,6 @@ public class MockupMapContext implements MapContext{
 throw new NotImplementedException();
 }
 
-@Override
 public IntegerRanges getProfileTaskRange(boolean isMap) {
 throw new NotImplementedException();
 }
@@ -318,4 +315,8 @@ public class MockupMapContext implements MapContext{
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+public boolean userClassesTakesPrecedence() {
+throw new NotImplementedException();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 

[11/47] kylin git commit: minor, fix false positive warning in test output

[billyliu]

minor, fix false positive warning in test output


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85a1eb39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85a1eb39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85a1eb39

Branch: refs/heads/KYLIN-2361
Commit: 85a1eb399d8dcd099d4607aa802faa5ccdf4d2bf
Parents: 1e4ae54
Author: Billy Liu 
Authored: Fri Jan 20 08:13:11 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 08:13:11 2017 +0800

--
 .../org/apache/kylin/common/util/AbstractKylinTestCase.java   | 7 ++-
 .../java/org/apache/kylin/dict/lookup/SnapshotManager.java| 6 ++
 .../src/main/java/org/apache/kylin/job/dao/ExecutableDao.java | 6 ++
 .../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 2 +-
 examples/test_case_data/localmeta/cube_desc/ssb.json  | 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube1.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube2.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube3.json| 4 ++--
 .../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 4 ++--
 .../localmeta/cube_desc/test_streaming_table_cube_desc.json   | 2 +-
 examples/test_case_data/localmeta/kylin.properties| 2 +-
 examples/test_case_data/sandbox/kylin.properties  | 2 +-
 kylin-it/pom.xml  | 2 +-
 13 files changed, 33 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 2154c32..4a9804d 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -31,12 +31,17 @@ public abstract class AbstractKylinTestCase {
 public static final String[] SERVICES_WITH_CACHE = { //
 "org.apache.kylin.cube.CubeManager", //
 "org.apache.kylin.cube.CubeDescManager", //
+"org.apache.kylin.dict.lookup.SnapshotManager", //
+"org.apache.kylin.dict.DictionaryManager", //
 "org.apache.kylin.storage.hybrid.HybridManager", //
 "org.apache.kylin.metadata.realization.RealizationRegistry", //
 "org.apache.kylin.metadata.project.ProjectManager", //
 "org.apache.kylin.metadata.MetadataManager", //
+"org.apache.kylin.metadata.cachesync.Broadcaster", //
+"org.apache.kylin.metadata.badquery.BadQueryHistoryManager", //
 "org.apache.kylin.job.impl.threadpool.DistributedScheduler", //
-"org.apache.kylin.job.manager.ExecutableManager", //
+"org.apache.kylin.job.execution.ExecutableManager", //
+"org.apache.kylin.job.dao.ExecutableDao" //
 };
 
 public abstract void createTestMetadata() throws Exception;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index 085158a..b45d017 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -66,6 +66,12 @@ public class SnapshotManager {
 return r;
 }
 
+public static void clearCache() {
+synchronized (SERVICE_CACHE) {
+SERVICE_CACHE.clear();
+}
+}
+
 // 

 
 private KylinConfig config;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java 
b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index 5cae5ac..96505e6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -64,6 +64,12 @@ public class ExecutableDao {
 return r;
 }
 
+public static void clearCache() {
+

[10/47] kylin git commit: KYLIN-2375 change default ehcache size to 256M

[billyliu]

KYLIN-2375 change default ehcache size to 256M


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e4ae540
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e4ae540
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e4ae540

Branch: refs/heads/KYLIN-2361
Commit: 1e4ae540bf194c7e67bb622c2eb0cd0ff1a6657c
Parents: 2196975
Author: shaofengshi 
Authored: Thu Jan 19 17:44:19 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:44:19 2017 +0800

--
 server/src/main/resources/ehcache-test.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e4ae540/server/src/main/resources/ehcache-test.xml
--
diff --git a/server/src/main/resources/ehcache-test.xml 
b/server/src/main/resources/ehcache-test.xml
index eb39774..bffe27a 100644
--- a/server/src/main/resources/ehcache-test.xml
+++ b/server/src/main/resources/ehcache-test.xml
@@ -12,7 +12,7 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
->
+>
 

[01/47] kylin git commit: KYLIN-2348 let choose model freely for sub-queries [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2361 f577a7da0 -> 9a3bd71c8 (forced update)


KYLIN-2348 let choose model freely for sub-queries


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d52bb8bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d52bb8bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d52bb8bc

Branch: refs/heads/KYLIN-2361
Commit: d52bb8bc05bd530362fe0970b8e8a85f8b2289bc
Parents: 51c6571
Author: Li Yang 
Authored: Wed Jan 18 22:46:24 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 18 22:46:24 2017 +0800

--
 .../apache/kylin/query/relnode/OLAPTableScan.java   |  2 ++
 .../apache/kylin/query/routing/ModelChooser.java| 16 
 2 files changed, 10 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d52bb8bc/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 452170a..f7877be 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -53,6 +53,7 @@ import org.apache.calcite.rel.rules.JoinPushExpressionsRule;
 import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
 import org.apache.calcite.rel.rules.JoinUnionTransposeRule;
 import org.apache.calcite.rel.rules.ReduceExpressionsRule;
+import org.apache.calcite.rel.rules.SemiJoinRule;
 import org.apache.calcite.rel.rules.SortJoinTransposeRule;
 import org.apache.calcite.rel.rules.SortUnionTransposeRule;
 import org.apache.calcite.rel.type.RelDataType;
@@ -174,6 +175,7 @@ public class OLAPTableScan extends TableScan implements 
OLAPRel, EnumerableRel {
 planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
 planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
 planner.removeRule(DateRangeRules.FILTER_INSTANCE);
+planner.removeRule(SemiJoinRule.INSTANCE);
 // distinct count will be split into a separated query that is joined 
with the left query
 planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d52bb8bc/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java 
b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index 2517dc5..f979f7b 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -53,17 +53,17 @@ public class ModelChooser {
 IdentityHashMap candidates = new 
IdentityHashMap<>();
 
 // attempt one model for all contexts
-Set reals = attemptSelectModel(contexts);
-if (reals != null) {
-for (OLAPContext ctx : contexts) {
-candidates.put(ctx, reals);
-}
-return candidates;
-}
+//Set reals = attemptSelectModel(contexts);
+//if (reals != null) {
+//for (OLAPContext ctx : contexts) {
+//candidates.put(ctx, reals);
+//}
+//return candidates;
+//}
 
 // try different model for different context
 for (OLAPContext ctx : contexts) {
-reals = attemptSelectModel(ImmutableList.of(ctx));
+Set reals = 
attemptSelectModel(ImmutableList.of(ctx));
 if (reals == null)
 throw new NoRealizationFoundException("No model found for" + 
toErrorMsg(ctx));
 

[06/47] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[billyliu]

KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dfb5fac9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dfb5fac9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dfb5fac9

Branch: refs/heads/KYLIN-2361
Commit: dfb5fac9f0d9c13f1128206b73e683b515fe3d72
Parents: 61833d9
Author: Billy Liu 
Authored: Thu Jan 19 14:54:20 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 19 14:54:20 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3941 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2430 insertions(+), 1571 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b916df2..1b327fe 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -36,7 +36,16 @@
 
 org.apache.calcite
 calcite-core
+
+
+org.apache.calcite.avatica
+avatica-core
+
+
+
+
+org.apache.calcite.avatica
+avatica
 
 
-
 

[02/47] kylin git commit: KYLIN-2410 let global dictionary use hadoop conf in mapper/reducer context

[billyliu]

KYLIN-2410 let global dictionary use hadoop conf in mapper/reducer context


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eddb695b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eddb695b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eddb695b

Branch: refs/heads/KYLIN-2361
Commit: eddb695ba13de282ada123f358aff723cb414316
Parents: d52bb8b
Author: Yang Li 
Authored: Thu Jan 19 08:56:22 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 19 08:56:22 2017 +0800

--
 core-common/pom.xml |   5 +
 .../apache/kylin/common/util/HadoopUtil.java| 146 ++
 .../apache/kylin/dict/AppendTrieDictionary.java |  13 +-
 .../kylin/dict/AppendTrieDictionaryChecker.java |  13 +-
 .../org/apache/kylin/dict/CachedTreeMap.java|   7 +-
 .../kylin/dict/AppendTrieDictionaryTest.java|   6 +-
 .../apache/kylin/dict/CachedTreeMapTest.java|   8 +-
 .../apache/kylin/engine/mr/DFSFileTable.java|   1 +
 .../kylin/engine/mr/DFSFileTableReader.java |   1 +
 .../org/apache/kylin/engine/mr/HadoopUtil.java  | 150 ---
 .../org/apache/kylin/engine/mr/KylinMapper.java |   1 +
 .../apache/kylin/engine/mr/KylinReducer.java|   1 +
 .../kylin/engine/mr/SortedColumnDFSFile.java|   1 +
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../kylin/engine/mr/common/CubeStatsReader.java |   2 +-
 .../engine/mr/common/MapReduceExecutable.java   |   2 +-
 .../engine/mr/steps/CreateDictionaryJob.java|   2 +-
 .../mr/steps/FactDistinctColumnsReducer.java|   2 +-
 .../kylin/engine/mr/steps/InMemCuboidJob.java   |   2 +-
 .../engine/mr/steps/MergeStatisticsStep.java|   2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  |   2 +-
 .../engine/mr/steps/SaveStatisticsStep.java |   2 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   2 +-
 .../steps/FactDistinctColumnsReducerTest.java   |   2 +-
 .../HiveToBaseCuboidMapperPerformanceTest.java  |   2 +-
 .../engine/mr/steps/MergeCuboidJobTest.java |   2 +-
 .../kylin/engine/mr/steps/NDCuboidJobTest.java  |   2 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |   2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |   2 +-
 .../kylin/provision/BuildCubeWithEngine.java|   2 +-
 .../apache/kylin/query/ITMassInQueryTest.java   |   2 +-
 .../kylin/storage/hbase/ITHdfsOpsTest.java  |   2 +-
 .../rest/controller/StreamingController.java|   2 +-
 .../apache/kylin/rest/service/ModelService.java |   2 +-
 .../apache/kylin/rest/service/TableService.java |   2 +-
 .../apache/kylin/source/hive/HiveMRInput.java   |   2 +-
 .../source/hive/HiveSourceTableLoader.java  |   2 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |   2 +-
 .../kylin/storage/hbase/HBaseConnection.java|   2 +-
 .../kylin/storage/hbase/HBaseResourceStore.java |   2 +-
 .../storage/hbase/steps/DeprecatedGCStep.java   |   2 +-
 .../steps/HDFSPathGarbageCollectionStep.java|   2 +-
 .../hbase/steps/SequenceFileCuboidWriter.java   |   2 +-
 .../storage/hbase/util/CubeMigrationCLI.java|   2 +-
 .../storage/hbase/util/StorageCleanupJob.java   |   2 +-
 .../storage/hbase/steps/CreateHTableTest.java   |   2 +-
 .../hbase/steps/CubeHFileMapper2Test.java   |   2 +-
 .../steps/RangeKeyDistributionJobTest.java  |   2 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java |   2 +-
 .../apache/kylin/tool/MrJobInfoExtractor.java   |   2 +-
 .../apache/kylin/tool/StorageCleanupJob.java|   2 +-
 .../org/apache/kylin/tool/util/ToolUtil.java|   2 +-
 52 files changed, 216 insertions(+), 215 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 25b10a7..016d470 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -78,6 +78,11 @@
 
 
 
+org.apache.hadoop
+hadoop-common
+provided
+
+
 junit
 junit
 test

http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
new file mode 100644
index 000..390c209
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work 

[05/47] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index e0782ce..171e299 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -16,24 +16,6 @@
  */
 package org.apache.calcite.sql2rel;
 
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
 import org.apache.calcite.avatica.util.Spaces;
 import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.plan.Convention;
@@ -85,7 +67,6 @@ import org.apache.calcite.rel.stream.Delta;
 import org.apache.calcite.rel.stream.LogicalDelta;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -116,6 +97,7 @@ import org.apache.calcite.sql.SqlCallBinding;
 import org.apache.calcite.sql.SqlDataTypeSpec;
 import org.apache.calcite.sql.SqlDelete;
 import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainFormat;
 import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.calcite.sql.SqlFunction;
 import org.apache.calcite.sql.SqlIdentifier;
@@ -177,17 +159,37 @@ import org.apache.calcite.util.NumberUtil;
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Util;
 import org.apache.calcite.util.trace.CalciteTrace;
-import org.slf4j.Logger;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+
+import org.slf4j.Logger;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
 
 /*
  * The code has synced with calcite. Hope one day, we could remove the 
hardcode override point.
@@ -209,42 +211,56 @@ import com.google.common.collect.Sets;
 public class SqlToRelConverter {
 //~ Static fields/initializers 
-
 
+protected static final Logger SQL2REL_LOGGER =
+CalciteTrace.getSqlToRelTracer();
+
+private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
 /** Size of the smallest IN list that will be converted to a semijoin to a
  * static table. */
-public static final int DEFAULT_IN_SUBQUERY_THRESHOLD = 20;
-protected static final Logger SQL2REL_LOGGER = 
CalciteTrace.getSqlToRelTracer();
-private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+
+@Deprecated // to be removed before 2.0
+public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
+DEFAULT_IN_SUB_QUERY_THRESHOLD;
 
 //~ Instance fields 

-public final SqlToRelConverter.Config config;
-public final RelOptTable.ViewExpander viewExpander;
+
 protected final SqlValidator validator;
 protected final RexBuilder rexBuilder;
 protected final Prepare.CatalogReader catalogReader;
 protected final RelOptCluster cluster;
+private DefaultValueFactory defaultValueFactory;
+private SubQueryConverter subQueryConverter;
 protected final List leaves = new ArrayList<>();
-protected final RelDataTypeFactory typeFactory;
 private final List dynamicParamSqlNodes = new 
ArrayList<>();
 

kylin git commit: KYLIN-2419 rollback KYLIN-2292

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master a058bfb8b -> 7611338b5


KYLIN-2419 rollback KYLIN-2292


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7611338b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7611338b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7611338b

Branch: refs/heads/master
Commit: 7611338b5f022d216f5c9564a13c161374751adf
Parents: a058bfb
Author: Li Yang 
Authored: Sat Feb 4 11:03:25 2017 +0800
Committer: Li Yang 
Committed: Sat Feb 4 11:03:25 2017 +0800

--
 .../adapter/enumerable/EnumerableWindow.java| 981 ---
 .../calcite/adapter/enumerable/PhysType.java| 209 
 .../adapter/enumerable/PhysTypeImpl.java| 654 -
 3 files changed, 1844 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7611338b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
 
b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
deleted file mode 100644
index 216b07c..000
--- 
a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
+++ /dev/null
@@ -1,981 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.adapter.enumerable;
-
-import org.apache.calcite.adapter.enumerable.impl.WinAggAddContextImpl;
-import org.apache.calcite.adapter.enumerable.impl.WinAggResetContextImpl;
-import org.apache.calcite.adapter.enumerable.impl.WinAggResultContextImpl;
-import org.apache.calcite.adapter.java.JavaTypeFactory;
-import org.apache.calcite.linq4j.tree.BinaryExpression;
-import org.apache.calcite.linq4j.tree.BlockBuilder;
-import org.apache.calcite.linq4j.tree.BlockStatement;
-import org.apache.calcite.linq4j.tree.DeclarationStatement;
-import org.apache.calcite.linq4j.tree.Expression;
-import org.apache.calcite.linq4j.tree.Expressions;
-import org.apache.calcite.linq4j.tree.ParameterExpression;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.linq4j.tree.Statement;
-import org.apache.calcite.linq4j.tree.Types;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.prepare.CalcitePrepareImpl;
-import org.apache.calcite.rel.RelFieldCollation;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.core.Window;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexWindowBound;
-import org.apache.calcite.runtime.SortedMultiMap;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.util.BuiltInMethod;
-import org.apache.calcite.util.Pair;
-import org.apache.calcite.util.Util;
-
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableList;
-
-import java.lang.reflect.Modifier;
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-
-/*
- * OVERRIDE POINT: patching CALCITE-1540 on calcite 1.8.0
- */
-
-/** Implementation of {@link org.apache.calcite.rel.core.Window} in
- * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention 
enumerable calling convention}. */
-public class EnumerableWindow extends Window implements EnumerableRel {
-/** Creates an EnumerableWindowRel. */
-EnumerableWindow(RelOptCluster cluster, RelTraitSet traits, RelNode child,
-List constants,