kylin git commit: KYLIN-2420 Record zookeeper lock holder information

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 7611338b5 -> 56c7aa505


KYLIN-2420 Record zookeeper lock holder information

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56c7aa50
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56c7aa50
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56c7aa50

Branch: refs/heads/master
Commit: 56c7aa505706f9aa878b4d3804cdf3f357a7931f
Parents: 7611338
Author: Roger Shi 
Authored: Sat Feb 4 13:43:03 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 13:46:59 2017 +0800

--
 .../storage/hbase/util/ZookeeperJobLock.java| 26 +++-
 1 file changed, 25 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56c7aa50/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
index bdd3981..7bf7498 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperJobLock.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.storage.hbase.util;
 
+import java.lang.management.ManagementFactory;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
 
@@ -35,6 +37,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.job.lock.JobLock;
 import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,9 +66,14 @@ public class ZookeeperJobLock implements JobLock {
 }
 
 RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
-this.zkClient = CuratorFrameworkFactory.newClient(zkConnectString, 
retryPolicy);
+try {
+this.zkClient = 
CuratorFrameworkFactory.builder().connectString(zkConnectString).retryPolicy(retryPolicy).defaultData(getIpProcess()).build();
+} catch (UnknownHostException e) {
+throw new RuntimeException(e);
+}
 this.zkClient.start();
 this.sharedLock = new InterProcessMutex(zkClient, this.scheduleID);
+
 boolean hasLock = false;
 try {
 hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
@@ -74,9 +82,20 @@ public class ZookeeperJobLock implements JobLock {
 }
 if (!hasLock) {
 logger.warn("fail to acquire lock, scheduler has not been started; 
maybe another kylin process is still running?");
+try {
+for (String node : sharedLock.getParticipantNodes()) {
+logger.warn("lock holder info: {}", new 
String(zkClient.getData().forPath(node)));
+}
+} catch (Exception e) {
+logger.warn("error check participant", e);
+if (!(e instanceof KeeperException.NoNodeException)) {
+throw new RuntimeException(e);
+}
+}
 zkClient.close();
 return false;
 }
+
 return true;
 }
 
@@ -115,4 +134,9 @@ public class ZookeeperJobLock implements JobLock {
 private String schedulerId() {
 return ZOOKEEPER_LOCK_PATH + "/" + 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
 }
+
+private byte[] getIpProcess() throws UnknownHostException {
+logger.info("get IP and processId: {}", 
ManagementFactory.getRuntimeMXBean().getName().getBytes());
+return ManagementFactory.getRuntimeMXBean().getName().getBytes();
+}
 }

[14/47] kylin git commit: KYLIN-2308 Allow user to set more columnFamily in web

[billyliu]

KYLIN-2308 Allow user to set more columnFamily in web

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8331d8d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8331d8d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8331d8d0

Branch: refs/heads/KYLIN-2361
Commit: 8331d8d0ad37d24bdbfa90d1032bcdd063aa0a59
Parents: a853a7c
Author: kangkaisen 
Authored: Sat Jan 7 15:34:57 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 17:25:28 2017 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 75 +++-
 webapp/app/js/controllers/cubeEdit.js   | 51 ---
 webapp/app/js/controllers/cubeSchema.js | 13 +++
 webapp/app/js/filters/filter.js | 22 +++--
 .../cubeDesigner/advanced_settings.html | 92 +++-
 5 files changed, 194 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 760133a..39d36b0 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -278,11 +278,84 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 $scope.isReuse=!$scope.isReuse;
   }
 
-  $scope.removeDictionaries =  function(arr,element){
+  $scope.removeElement =  function(arr,element){
 var index = arr.indexOf(element);
 if (index > -1) {
   arr.splice(index, 1);
 }
   };
 
+  $scope.newColFamily = function (index) {
+return {
+"name": "F" + index,
+"columns": [
+  {
+"qualifier": "M",
+"measure_refs": []
+  }
+]
+  };
+  };
+
+  $scope.initColumnFamily = function () {
+$scope.cubeMetaFrame.hbase_mapping.column_family = [];
+var normalMeasures = [], distinctCountMeasures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  if (measure.function.expression === 'COUNT_DISTINCT') {
+distinctCountMeasures.push(measure);
+  } else {
+normalMeasures.push(measure);
+  }
+});
+if (normalMeasures.length > 0) {
+  var nmcf = $scope.newColFamily(1);
+  angular.forEach(normalMeasures, function (normalM, index) {
+nmcf.columns[0].measure_refs.push(normalM.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(nmcf);
+}
+
+if (distinctCountMeasures.length > 0) {
+  var dccf = $scope.newColFamily(2);
+  angular.forEach(distinctCountMeasures, function (dcm, index) {
+dccf.columns[0].measure_refs.push(dcm.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(dccf);
+}
+  };
+
+  $scope.getAllMeasureNames = function () {
+var measures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  measures.push(measure.name);
+});
+return measures;
+  };
+
+  $scope.getAssignedMeasureNames = function () {
+var assignedMeasures = [];
+angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function 
(colFamily, index) {
+  angular.forEach(colFamily.columns[0].measure_refs, function (measure, 
index) {
+assignedMeasures.push(measure);
+  });
+});
+return assignedMeasures;
+  };
+
+  if ($scope.getAllMeasureNames().length != 
$scope.getAssignedMeasureNames().length) {
+$scope.initColumnFamily();
+  }
+
+
+  $scope.addColumnFamily = function () {
+var colFamily = 
$scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 
1);
+$scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
+  };
+
+  $scope.refreshColumnFamily = function (column_familys, index, colFamily) {
+if (column_familys) {
+  column_familys[index] = colFamily;
+}
+  };
+
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index edbb421..da19b22 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -284,25 +284,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return type;
   };
 
-  var ColFamily = function () {
-var index = 1;
-return function () {
-  var newColFamily =
-  {
-"name": "f" + index,
-"columns": [
-  {
-

[04/47] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index cd784f4..874ead6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -66,13 +66,15 @@
 
 
 
false
+
 
-
 
-
com.fasterxml.jackson.core:*
+com.google.protobuf:*
+commons-logging:*
+commons-codec:*
 
 
+
 
 
 org.apache.calcite
@@ -86,10 +88,6 @@
 org.apache.http
 
${shadeBase}.org.apache.http
 
-
-org.apache.commons
-
${shadeBase}.org.apache.commons
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index a1b9aef..8e69e68 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -209,7 +209,7 @@ public class KylinMeta extends MetaImpl {
 } catch (NoSuchFieldException e) {
 throw new RuntimeException(e);
 }
-columns.add(columnMetaData(name, index, field.getType()));
+columns.add(columnMetaData(name, index, field.getType(), true));
 fields.add(field);
 fieldNames.add(fieldName);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 080558b..80af108 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -43,6 +43,12 @@
 
 org.apache.kylin
 atopcalcite
+
+
+avatica-core
+org.apache.calcite.avatica
+
+
 
 
 org.apache.kylin
@@ -74,14 +80,6 @@
 org.apache.kylin
 kylin-query
 
-
-org.apache.calcite
-calcite-linq4j
-
-
-commons-cli
-commons-cli
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java 
b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index df6eb2f..05f615f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
 import java.io.File;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
+import java.sql.Driver;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.Statement;

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/pom.xml
--
diff --git a/pom.xml b/pom.xml
index ff4c4e8..57b7752 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,8 +112,8 @@
 1.8.9
 
 
-1.10.0
-1.8.0
+1.11.0
+1.9.0
 
 
 jacoco
@@ -450,15 +450,9 @@
 
 
 
-org.apache.calcite
-calcite-linq4j
-${calcite.version}
-
-
 org.apache.calcite.avatica
 avatica
 ${avatica.version}
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/query/pom.xml
--
diff --git a/query/pom.xml b/query/pom.xml
index 1dc05d1..6ab74a7 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -36,16 +36,18 @@
 
 org.apache.kylin
 atopcalcite
+
+
+org.apache.calcite.avatica
+avatica-core
+
+

[07/47] kylin git commit: KYLIN-2387 code refactor, merge ImmutableBitmapCounter and MutableBitmapCounter into RoaringBitmapCounter

[billyliu]

KYLIN-2387 code refactor, merge ImmutableBitmapCounter and MutableBitmapCounter 
into RoaringBitmapCounter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38c3e7bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38c3e7bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38c3e7bf

Branch: refs/heads/KYLIN-2361
Commit: 38c3e7bf691ecdfd0f8d42fcc97065a0596be018
Parents: 61833d9
Author: gaodayue 
Authored: Thu Jan 19 15:23:34 2017 +0800
Committer: gaodayue 
Committed: Thu Jan 19 15:43:14 2017 +0800

--
 .../gridtable/AggregationCacheMemSizeTest.java  |  11 +-
 .../metadata/measure/MeasureCodecTest.java  |   5 +-
 .../kylin/measure/bitmap/BitmapAggregator.java  |  25 +---
 .../kylin/measure/bitmap/BitmapCounter.java |  28 +++-
 .../measure/bitmap/BitmapCounterFactory.java|  30 
 .../BitmapIntersectDistinctCountAggFunc.java|  16 +-
 .../kylin/measure/bitmap/BitmapMeasureType.java |  17 ++-
 .../kylin/measure/bitmap/BitmapSerializer.java  |   9 +-
 .../measure/bitmap/ImmutableBitmapCounter.java  | 108 --
 .../measure/bitmap/MutableBitmapCounter.java|  60 
 .../measure/bitmap/RoaringBitmapCounter.java| 147 +++
 .../bitmap/RoaringBitmapCounterFactory.java |  47 ++
 .../measure/AggregatorMemEstimateTest.java  |   5 +-
 .../measure/bitmap/BitmapAggregatorTest.java|  56 ++-
 .../kylin/measure/bitmap/BitmapCounterTest.java |  32 +---
 .../measure/bitmap/BitmapSerializerTest.java|   5 +-
 16 files changed, 345 insertions(+), 256 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 63c7672..f749fb4 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -33,7 +33,9 @@ import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
 import org.apache.kylin.measure.basic.DoubleSumAggregator;
 import org.apache.kylin.measure.basic.LongSumAggregator;
 import org.apache.kylin.measure.bitmap.BitmapAggregator;
-import org.apache.kylin.measure.bitmap.MutableBitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounter;
+import org.apache.kylin.measure.bitmap.BitmapCounterFactory;
+import org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory;
 import org.apache.kylin.measure.hllc.HLLCAggregator;
 import org.apache.kylin.measure.hllc.HLLCounter;
 import org.github.jamm.MemoryMeter;
@@ -43,13 +45,14 @@ import com.google.common.base.Stopwatch;
 
 public class AggregationCacheMemSizeTest {
 private static final MemoryMeter meter = new MemoryMeter();
-private static final MutableBitmapCounter[] bitmaps = new 
MutableBitmapCounter[5];
+private static final BitmapCounterFactory bitmapFactory = 
RoaringBitmapCounterFactory.INSTANCE;
+private static final BitmapCounter[] bitmaps = new BitmapCounter[5];
 private static final Random random = new Random();
 
 // consider bitmaps with variant cardinality
 static {
 for (int i = 0; i < bitmaps.length; i++) {
-bitmaps[i] = new MutableBitmapCounter();
+bitmaps[i] = bitmapFactory.newBitmap();
 }
 
 final int totalBits = 1_000_000;
@@ -116,7 +119,7 @@ public class AggregationCacheMemSizeTest {
 }
 
 private BitmapAggregator createBitmapAggr(boolean lowCardinality) {
-MutableBitmapCounter counter = new MutableBitmapCounter();
+BitmapCounter counter = bitmapFactory.newBitmap();
 counter.orWith(lowCardinality ? bitmaps[0] : bitmaps[3]);
 
 BitmapAggregator result = new BitmapAggregator();

http://git-wip-us.apache.org/repos/asf/kylin/blob/38c3e7bf/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
--
diff --git 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 97c9751..7129a5e 100644
--- 
a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ 
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -25,7 +25,8 @@ import java.nio.ByteBuffer;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import 

[31/47] kylin git commit: KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs

[billyliu]

KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6dda4fe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6dda4fe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6dda4fe

Branch: refs/heads/KYLIN-2361
Commit: f6dda4fecd5a882a17379987445a5aa0a347ff63
Parents: 4c33e38
Author: kangkaisen 
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 ++-
 .../mr/steps/FactDistinctColumnsReducer.java| 240 +++
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 175 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
 }
 }
 
+public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+@Override
+public boolean accept(Path path) {
+return path.getName().startsWith(filter);
+}
+});
+
+if (fileStatus.length == 1) {
+return fileStatus[0].getPath();
+} else {
+return null;
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/statistics";
+return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/f6dda4fe/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
-String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
+String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+String CFG_OUTPUT_COLUMN = "column";
+String CFG_OUTPUT_DICT = "dict";
+String 

[36/47] kylin git commit: minor, upgrade default hdp version from 2.2.4.2-2 to 2.4.0.0-169

[billyliu]

minor, upgrade default hdp version from 2.2.4.2-2 to 2.4.0.0-169


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/571b7914
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/571b7914
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/571b7914

Branch: refs/heads/KYLIN-2361
Commit: 571b7914f763e05160fb7598bbc2bfbe54d8c7cf
Parents: b6d247c
Author: Hongbin Ma 
Authored: Tue Jan 24 15:29:05 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Jan 24 15:29:10 2017 +0800

--
 .../java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java   | 4 ++--
 .../java/org/apache/kylin/provision/BuildCubeWithEngine.java | 4 ++--
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java | 2 +-
 server/src/main/java/org/apache/kylin/rest/DebugTomcat.java  | 2 +-
 .../apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java| 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
index 03509cb..dac6aa9 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
@@ -46,7 +46,7 @@ public class DeployLocalMetaToRemoteTest {
 ClassUtil.addClasspath(new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
 System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
-throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
 }
 }
 
@@ -70,4 +70,4 @@ public class DeployLocalMetaToRemoteTest {
 System.out.println("blank");
 }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index cbec4f6..08cc6b9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -112,7 +112,7 @@ public class BuildCubeWithEngine {
 
 System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
-throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
 }
 
 
HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
@@ -363,4 +363,4 @@ public class BuildCubeWithEngine {
 }
 }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/571b7914/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 194ea35..37ff8ae 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -274,7 +274,7 @@ public class BuildCubeWithStream {
 ClassUtil.addClasspath(new 
File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
 System.setProperty(KylinConfig.KYLIN_CONF, 
HBaseMetadataTestCase.SANDBOX_TEST_DATA);
 if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
-throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+throw new RuntimeException("No hdp.version set; Please set 
hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
 

[32/47] kylin git commit: KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase

[billyliu]

KYLIN-2374 Allow kylin to store metadata in HDFS instead of HBase

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d23bf930
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d23bf930
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d23bf930

Branch: refs/heads/KYLIN-2361
Commit: d23bf930da0b542d0e6981917e6bde055839577a
Parents: db85d66
Author: xiefan46 <958034...@qq.com>
Authored: Wed Jan 11 10:00:19 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   5 +
 .../common/persistence/ResourceStoreTest.java   |   3 +-
 .../test_case_data/sandbox/kylin.properties |   1 +
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 117 +++
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 +
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 198 +++
 7 files changed, 409 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 05df177..44d636d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -201,6 +201,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metadata.url");
 }
 
+//for hdfs resource store
+public String getHDFSMetadataUrl() {
+return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
+}
+
 // for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);

http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 4c31a15..ddaf481 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -110,9 +110,10 @@ public class ResourceStoreTest {
 }
 
 // list
-NavigableSet list;
+NavigableSet list = null;
 
 list = store.listResources(dir1);
+System.out.println(list);
 assertTrue(list.contains(path1));
 assertTrue(list.contains(path2) == false);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 6c512dc..b01c377 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -41,6 +41,7 @@ kylin.source.hive.client=cli
 # The metadata store in hbase
 kylin.metadata.url=kylin_default_instance@hbase
 
+
 # The storage for final cube file in hbase
 kylin.storage.url=hbase
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d23bf930/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
new file mode 100644
index 000..ef04957
--- /dev/null
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under 

[24/47] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
 FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 String localCoprocessorJar;
 if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
 public static void deployCoprocessor(HTableDescriptor tableDesc) {
 try {
 initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with 
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed 
with coprocessor.");
 
 } catch (Exception ex) {
-logger.error("Error deploying coprocessor on " + 
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " + 
tableDesc.getTableName(), ex);
 logger.error("Will try creating the table without coprocessor.");
 }
 }
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
 desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
 }
 
-public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, 
Path hdfsCoprocessorJar) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 HTableDescriptor desc = 
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
 
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
 logger.info("reset coprocessor on " + tableName);
 
 logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
 
 while (desc.hasCoprocessor(CubeObserverClassOld2)) {
 desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
 desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
 }
 
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
 
 logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
 
 return true;
 }
 
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
+private static List resetCoprocessorOnHTables(final Admin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
 List processedTables = Collections.synchronizedList(new 
ArrayList());
 ExecutorService coprocessorPool = 
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
 CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
 
 private static class ResetCoprocessorWorker implements Runnable {
 private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
 private final Path hdfsCoprocessorJar;
 private final String tableName;
 private final List processedTables;
 
-public ResetCoprocessorWorker(CountDownLatch countDownLatch, 
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List 
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch 

[25/47] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/169c3de6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/169c3de6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/169c3de6

Branch: refs/heads/KYLIN-2361
Commit: 169c3de6456f394d049e9550d65a98f6aa0a316f
Parents: 68e87f0
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 13:01:20 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/169c3de6/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index e02bf19..cbec4f6 100644
--- 

[12/47] kylin git commit: KYLIN-2394 merge CALCITE-1592

[billyliu]

KYLIN-2394 merge CALCITE-1592


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34f2d3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34f2d3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34f2d3e

Branch: refs/heads/KYLIN-2361
Commit: d34f2d3e2bf691c55077a9a76a89217ae915bbdd
Parents: dfb5fac
Author: Billy Liu 
Authored: Fri Jan 20 15:42:37 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 15:42:37 2017 +0800

--
 .../calcite/sql2rel/SqlToRelConverter.java  | 51 +---
 1 file changed, 23 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d34f2d3e/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 171e299..cc9c9a1 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -935,19 +935,21 @@ public class SqlToRelConverter {
 /**
  * Push down all the NOT logical operators into any IN/NOT IN operators.
  *
+ * @param scope Scope where {@code sqlNode} occurs
  * @param sqlNode the root node from which to look for NOT operators
  * @return the transformed SqlNode representation with NOT pushed down.
  */
-private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+private static SqlNode pushDownNotForIn(SqlValidatorScope scope,
+SqlNode sqlNode) {
 if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
 SqlCall sqlCall = (SqlCall) sqlNode;
 if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
 || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
 SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
 for (int i = 0; i < sqlOperands.length; i++) {
-sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+sqlOperands[i] = pushDownNotForIn(scope, sqlOperands[i]);
 }
-return sqlNode;
+return reg(scope, sqlNode);
 } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
 SqlNode childNode = sqlCall.operand(0);
 assert childNode instanceof SqlCall;
@@ -956,48 +958,34 @@ public class SqlToRelConverter {
 SqlNode[] andOperands = childSqlCall.getOperands();
 SqlNode[] orOperands = new SqlNode[andOperands.length];
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-andOperands[i]);
+orOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]));
 }
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] = pushDownNotForIn(orOperands[i]);
+orOperands[i] = pushDownNotForIn(scope, orOperands[i]);
 }
-return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
-orOperands[0], orOperands[1]);
+return reg(scope, 
SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], 
orOperands[1]));
 } else if (childSqlCall.getOperator() == 
SqlStdOperatorTable.OR) {
 SqlNode[] orOperands = childSqlCall.getOperands();
 SqlNode[] andOperands = new SqlNode[orOperands.length];
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-orOperands[i]);
+andOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]));
 }
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] = pushDownNotForIn(andOperands[i]);
+andOperands[i] = pushDownNotForIn(scope, 
andOperands[i]);
 }
-return 
SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
- 

[16/47] kylin git commit: KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs

[billyliu]

KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7de8aa12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7de8aa12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7de8aa12

Branch: refs/heads/KYLIN-2361
Commit: 7de8aa1203a72bad105ed692f7100535939b03af
Parents: c2229c9
Author: kangkaisen 
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: kangkaisen 
Committed: Sat Jan 21 23:19:50 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 ++-
 .../mr/steps/FactDistinctColumnsReducer.java| 240 +++
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 175 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
 }
 }
 
+public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+@Override
+public boolean accept(Path path) {
+return path.getName().startsWith(filter);
+}
+});
+
+if (fileStatus.length == 1) {
+return fileStatus[0].getPath();
+} else {
+return null;
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/statistics";
+return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
-String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
+String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+String CFG_OUTPUT_COLUMN = "column";
+String CFG_OUTPUT_DICT = "dict";
+String CFG_OUTPUT_STATISTICS = 

[08/47] kylin git commit: KYLIN-2409 change inmem cubing to single thread by default

[billyliu]

KYLIN-2409 change inmem cubing to single thread by default


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21969753
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21969753
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21969753

Branch: refs/heads/KYLIN-2361
Commit: 21969753cc66efa22bc9fc933af46346e9846631
Parents: b49c9e3
Author: shaofengshi 
Authored: Thu Jan 19 11:52:17 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:37:42 2017 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java  | 6 +-
 .../kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java| 2 +-
 .../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 9 +
 .../org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java | 2 ++
 4 files changed, 17 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d6774ff..74903d5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -300,6 +300,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
 }
 
+public int getCubeAlgorithmInMemConcurrentThreads() {
+return 
Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-concurrent-threads", 
"1"));
+}
+
 public boolean isIgnoreCubeSignatureInconsistency() {
 return 
Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", 
"false"));
 }
@@ -744,7 +748,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getYarnStatusCheckIntervalSeconds() {
-return 
Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", 
"60"));
+return 
Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", 
"10"));
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index 651203a..c7a4a05 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -46,7 +46,7 @@ abstract public class AbstractInMemCubeBuilder {
 final protected CubeDesc cubeDesc;
 final protected Map dictionaryMap;
 
-protected int taskThreadCount = 4;
+protected int taskThreadCount = 1;
 protected int reserveMemoryMB = 100;
 
 public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc 
flatDesc, Map dictionaryMap) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/21969753/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 55c04c6..9158aa3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -36,6 +36,7 @@ import java.util.Map.Entry;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.common.util.ImmutableBitSet;
 import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.common.util.Pair;
@@ -178,11 +179,16 @@ public class GTAggregateScanner implements IGTScanner {
 final List dumps;
 final int keyLength;
 final boolean[] compareMask;
+boolean compareAll = true;
 final BufferedMeasureCodec measureCodec;
 
 final Comparator bytesComparator = new Comparator() {
 @Override
 public int compare(byte[] o1, byte[] o2) {
+

[44/47] kylin git commit: KYLIN-2418 drop unused useSandbox

[billyliu]

KYLIN-2418 drop unused useSandbox


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/546f88f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/546f88f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/546f88f3

Branch: refs/heads/KYLIN-2361
Commit: 546f88f38ca017773c17ccaf2cf4812b0c7c4666
Parents: 0582512
Author: Yang Li 
Authored: Mon Jan 30 09:07:29 2017 +0800
Committer: Yang Li 
Committed: Tue Jan 31 08:11:41 2017 +0800

--
 .../kylin/common/persistence/FileResourceStore.java   |  4 
 .../org/apache/kylin/common/persistence/ResourceTool.java | 10 ++
 kylin-it/pom.xml  |  6 --
 pom.xml   |  8 
 .../apache/kylin/storage/hbase/HBaseResourceStore.java|  5 +
 5 files changed, 15 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index dcd3f38..3e012f5 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -160,4 +160,8 @@ public class FileResourceStore extends ResourceStore {
 return new File(root, resPath);
 }
 
+@Override
+public String toString() {
+return root.getAbsolutePath();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index b3aac09..6ba68ae 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -133,22 +133,24 @@ public class ResourceTool {
 ResourceStore src = ResourceStore.getStore(srcConfig);
 ResourceStore dst = ResourceStore.getStore(dstConfig);
 
+logger.info("Copy from {} to {}", src, dst);
+
 copyR(src, dst, path);
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, 
List paths) throws IOException {
 ResourceStore src = ResourceStore.getStore(srcConfig);
 ResourceStore dst = ResourceStore.getStore(dstConfig);
+
+logger.info("Copy from {} to {}", src, dst);
+
 for (String path : paths) {
 copyR(src, dst, path);
 }
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) 
throws IOException {
-
-ResourceStore src = ResourceStore.getStore(srcConfig);
-ResourceStore dst = ResourceStore.getStore(dstConfig);
-copyR(src, dst, "/");
+copy(srcConfig, dstConfig, "/");
 }
 
 public static void copyR(ResourceStore src, ResourceStore dst, String 
path) throws IOException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/546f88f3/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index f88db9f..9662806 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -272,10 +272,6 @@
 
 
 
-useSandbox
-true
-
-
 log4j.configuration
 
file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
 
@@ -298,7 +294,6 @@
 test
 java
 
--DuseSandbox=true
 
-Dhdp.version=${hdp.version}
 
-DfastBuildMode=${fastBuildMode}
 
-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties
@@ -320,7 +315,6 @@
 test
 java

[33/47] kylin git commit: Add Zookeeper Lock

[billyliu]

Add Zookeeper Lock

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d3276e2e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d3276e2e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d3276e2e

Branch: refs/heads/KYLIN-2361
Commit: d3276e2e909d3001724ee8fda1304ae8b7f08c63
Parents: d23bf93
Author: xiefan46 <958034...@qq.com>
Authored: Fri Jan 20 09:48:17 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  22 +-
 .../test_case_data/sandbox/kylin.properties |   4 +
 .../storage/hdfs/ITHDFSResourceStoreTest.java   |  66 +-
 .../kylin/storage/hdfs/ITLockManagerTest.java   | 205 +++
 .../kylin/storage/hbase/HBaseResourceStore.java |   1 +
 .../org/apache/kylin/storage/hdfs/HDFSLock.java |  41 
 .../kylin/storage/hdfs/HDFSLockManager.java |  45 
 .../kylin/storage/hdfs/HDFSResourceStore.java   |  95 +++--
 .../apache/kylin/storage/hdfs/LockManager.java  | 116 +++
 .../apache/kylin/storage/hdfs/ResourceLock.java |  51 +
 10 files changed, 471 insertions(+), 175 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 44d636d..75b38ff 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,6 +193,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
+public String getRawHdfsWorkingDirectory() {
+String root = getRequired("kylin.env.hdfs-working-dir");
+if (!root.endsWith("/")) {
+root += "/";
+}
+return root;
+}
+
 // 

 // METADATA
 // 

@@ -201,11 +209,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.metadata.url");
 }
 
-//for hdfs resource store
-public String getHDFSMetadataUrl() {
-return getOptional("kylin.metadata.hdfs.url", 
"kylin_default_instance_hdfs@hdfs");
-}
-
 // for test only
 public void setMetadataUrl(String metadataUrl) {
 setProperty("kylin.metadata.url", metadataUrl);
@@ -925,4 +928,13 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
 }
 
+//zoo keeper
+public String getZooKeeperHost() {
+return getOptional("kylin.storage-zookeeper.host", "localhost");
+}
+
+public String getZooKeeperPort() {
+return getOptional("kylin.storage-zookeeper.port", "2181");
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index b01c377..2c2da91 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -177,3 +177,7 @@ 
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
 #kylin.engine.spark-conf.spark.yarn.queue=default
 
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
+
+
+#zoo keeper
+kylin.storage-zookeeper.host=sandbox
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/d3276e2e/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index ef04957..27d8a3c 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ 

[46/47] kylin git commit: KYLIN-2361 Upgrade Tomcat 8.5.9

[billyliu]

KYLIN-2361 Upgrade Tomcat 8.5.9


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b60ac6a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b60ac6a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b60ac6a

Branch: refs/heads/KYLIN-2361
Commit: 2b60ac6a42741ca70d63e6680a0fbe9aeed7d46e
Parents: a058bfb
Author: Billy Liu 
Authored: Wed Jan 18 17:19:04 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 11:42:02 2017 +0800

--
 build/script/download-tomcat.sh |  8 
 pom.xml |  2 +-
 .../java/org/apache/kylin/rest/DebugTomcat.java | 16 +---
 .../kylin/ext/CustomizedWebappClassloader.java  |  5 ++---
 4 files changed, 16 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/build/script/download-tomcat.sh
--
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index b3aa509..bdfe351 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -27,19 +27,19 @@ if [[ `uname -a` =~ "Darwin" ]]; then
 alias md5cmd="md5 -q"
 fi
 
-tomcat_pkg_version="7.0.69"
-tomcat_pkg_md5="10a071e5169a1a8b14ff35a0ad181052"
+tomcat_pkg_version="8.5.9"
+tomcat_pkg_md5="b41270a64b7774c964e4bec813eea2ed"
 
 if [ ! -f "build/apache-tomcat-${tomcat_pkg_version}.tar.gz" ]
 then
 echo "no binary file found"
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "Download tomcat failed"
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "Download tomcat failed"
 else
 if [ `md5cmd build/apache-tomcat-${tomcat_pkg_version}.tar.gz | awk 
'{print $1}'` != "${tomcat_pkg_md5}" ]
 then
 echo "md5 check failed"
 rm build/apache-tomcat-${tomcat_pkg_version}.tar.gz
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "download tomcat failed"
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "download tomcat failed"
 fi
 fi
 unalias md5cmd

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index bf33e07..b82eee2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@
 3.2.4
 2.4.0
 2.5
-7.0.69
+8.5.9
 3.1
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
--
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java 
b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 3461e1d..1b47f79 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -21,11 +21,13 @@ package org.apache.kylin.rest;
 import org.apache.catalina.Context;
 import org.apache.catalina.core.AprLifecycleListener;
 import org.apache.catalina.core.StandardServer;
-import org.apache.catalina.deploy.ErrorPage;
 import org.apache.catalina.startup.Tomcat;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.tomcat.JarScanFilter;
+import org.apache.tomcat.JarScanType;
+import org.apache.tomcat.util.descriptor.web.ErrorPage;
 
 import java.io.File;
 import java.lang.reflect.Field;
@@ -127,12 +129,12 @@ public class DebugTomcat {
 notFound.setLocation("/index.html");
 webContext.addErrorPage(notFound);
 webContext.addWelcomeFile("index.html");
-//webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
-//@Override
-//public boolean check(JarScanType arg0, String arg1) {
-//return false;
-//}
-//});
+webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
+@Override
+public boolean check(JarScanType arg0, String arg1) {
+return false;
+}
+});
 
 // tomcat start
 tomcat.start();

[29/47] kylin git commit: KYLIN-2217 use column’s identity as path name and add more logging info

[billyliu]

KYLIN-2217 use column’s identity as path name and add more logging info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e217ae84
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e217ae84
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e217ae84

Branch: refs/heads/KYLIN-2361
Commit: e217ae84e1bd163658176093de208fcdf3bdf43c
Parents: f6dda4f
Author: shaofengshi 
Authored: Sun Jan 22 09:43:10 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../java/org/apache/kylin/common/util/HadoopUtil.java|  4 
 .../apache/kylin/cube/cli/DictionaryGeneratorCLI.java|  3 +++
 .../org/apache/kylin/engine/mr/SortedColumnDFSFile.java  |  4 
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java |  2 +-
 .../kylin/engine/mr/steps/CreateDictionaryJob.java   |  9 +++--
 .../engine/mr/steps/FactDistinctColumnsReducer.java  | 11 +++
 .../engine/mr/steps/UpdateCubeInfoAfterBuildStep.java|  4 ++--
 .../hive/cardinality/HiveColumnCardinalityJob.java   |  3 +++
 8 files changed, 31 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..f242515 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -143,6 +143,10 @@ public class HadoopUtil {
 }
 
 public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+if (fs.exists(baseDir) == false) {
+return null;
+}
+
 FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
 @Override
 public boolean accept(Path path) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 3e1ab0d..2e5a38e 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -63,11 +63,14 @@ public class DictionaryGeneratorCLI {
 if (dictProvider != null) {
 Dictionary dict = dictProvider.getDictionary(col);
 if (dict != null) {
+logger.debug("Dict for '" + col.getName() + "' has already 
been built, save it");
 cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);
 } else {
+logger.debug("Dict for '" + col.getName() + "' not 
pre-built, build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 } else {
+logger.debug("Dict for '" + col.getName() + "' not pre-built, 
build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
index d3f5cdc..f396b5a 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
@@ -128,4 +128,8 @@ public class SortedColumnDFSFile implements ReadableTable {
 return comparator;
 }
 
+@Override
+public String toString() {
+return dfsPath;
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e217ae84/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 567c1d0..44686d6 100644
--- 

[18/47] kylin git commit: KYLIN-2412 Unclosed DataOutputStream in RoaringBitmapCounter#write()

[billyliu]

KYLIN-2412 Unclosed DataOutputStream in RoaringBitmapCounter#write()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d264339b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d264339b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d264339b

Branch: refs/heads/KYLIN-2361
Commit: d264339b1c16c195ffafc2217b793d81bdbd6434
Parents: 7576a09
Author: gaodayue 
Authored: Sun Jan 22 14:02:09 2017 +0800
Committer: gaodayue 
Committed: Sun Jan 22 14:02:09 2017 +0800

--
 .../org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java| 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d264339b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index cd07d20..fb9dcfc 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -107,7 +107,9 @@ public class RoaringBitmapCounter implements BitmapCounter {
 if (out.remaining() < bitmap.serializedSizeInBytes()) {
 throw new BufferOverflowException();
 }
-bitmap.serialize(new DataOutputStream(new 
ByteBufferOutputStream(out)));
+try (DataOutputStream dos = new DataOutputStream(new 
ByteBufferOutputStream(out))) {
+bitmap.serialize(dos);
+}
 }
 
 @Override

[23/47] kylin git commit: add back calcite linq4j

[billyliu]

add back calcite linq4j


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68e87f08
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68e87f08
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68e87f08

Branch: refs/heads/KYLIN-2361
Commit: 68e87f08255caf33dd7946f43b3393372a73c68b
Parents: b56bc62
Author: Hongbin Ma 
Authored: Mon Jan 23 11:54:54 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 11:54:54 2017 +0800

--
 pom.xml | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/68e87f08/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 57b7752..e590fd7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -450,6 +450,11 @@
 
 
 
+org.apache.calcite
+calcite-linq4j
+${calcite.version}
+
+
 org.apache.calcite.avatica
 avatica
 ${avatica.version}

[39/47] kylin git commit: KYLIN-2374 minor, rename zookeeper config properties

[billyliu]

KYLIN-2374 minor,rename zookeeper config properties

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2a59c43
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2a59c43
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2a59c43

Branch: refs/heads/KYLIN-2361
Commit: a2a59c4303a29de308beb7b5c8300f78023313dd
Parents: 5629341
Author: xiefan46 <958034...@qq.com>
Authored: Tue Jan 24 18:57:07 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Jan 25 14:21:37 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java  | 4 ++--
 examples/test_case_data/sandbox/kylin.properties| 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a2a59c43/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 07cb124..5932197 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -930,11 +930,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 //zoo keeper
 public String getZooKeeperHost() {
-return getOptional("kylin.storage-zookeeper.host", "localhost");
+return getOptional("kylin.storage.lock-manager-zk-host", "localhost");
 }
 
 public String getZooKeeperPort() {
-return getOptional("kylin.storage-zookeeper.port", "2181");
+return getOptional("kylin.storage.lock-manager-zk-port", "2181");
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a2a59c43/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index fa0c6bc..6cb5148 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -55,6 +55,9 @@ kylin.env.hdfs-working-dir=/kylin
 
 kylin.engine.mr.reduce-input-mb=500
 
+#zoo keeper host in lock manager
+kylin.storage.lock-manager-zk-host=sandbox
+
 ### JOB ###
 
 # max job retry on error, default 0: no retry
@@ -179,5 +182,3 @@ 
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
 
 
-#zoo keeper
-kylin.storage-zookeeper.host=sandbox
\ No newline at end of file

[41/47] kylin git commit: KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217

[billyliu]

KYLIN-2414 Distinguish UHC columns from normal columns in KYLIN-2217

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6a9382b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6a9382b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6a9382b

Branch: refs/heads/KYLIN-2361
Commit: e6a9382b8d357e5bb13be2f678ebabb8bad75dc7
Parents: 5eae37e
Author: xiefan46 <958034...@qq.com>
Authored: Tue Jan 24 11:14:40 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 09:56:54 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 3 ---
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java  | 6 ++
 2 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b1acbbf..b25bcc0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -741,9 +741,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public boolean isReducerLocalBuildDict() {
-if (getUHCReducerCount() != 1) {
-return false;
-}
 return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a9382b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8c56bdf..cf94b30 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -132,6 +132,12 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
+int[] uhcIndex = 
CubeManager.getInstance(config).getUHCIndex(cubeDesc);
+int colIndex = reducerIdToColumnIndex.get(taskId);
+if (uhcIndex[colIndex] == 1)
+isReducerLocalBuildDict = false; //for UHC columns, this 
feature should be disabled
+}
 if (isReducerLocalBuildDict) {
 builder = 
DictionaryGenerator.newDictionaryBuilder(col.getType());
 builder.init(null, 0);

[27/47] kylin git commit: KYLIN-2413 update test conf files

[billyliu]

KYLIN-2413 update test conf files


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87d10394
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87d10394
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87d10394

Branch: refs/heads/KYLIN-2361
Commit: 87d103948fdf8951941c9a7fd11213dd569f8aa5
Parents: eb73da7
Author: lidongsjtu 
Authored: Mon Jan 23 13:12:36 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 13:18:24 2017 +0800

--
 dev-support/sync_hbase_cdh_branches.sh  |  15 +--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |  25 
 dev-support/test_all_against_hdp_2_4_0_0_169.sh |  25 
 .../sandbox/capacity-scheduler.xml  |  17 +--
 examples/test_case_data/sandbox/core-site.xml   |  35 -
 .../test_case_data/sandbox/hadoop-policy.xml|   2 +-
 .../test_case_data/sandbox/hbase-policy.xml |   2 +-
 examples/test_case_data/sandbox/hbase-site.xml  | 100 +--
 examples/test_case_data/sandbox/hdfs-site.xml   |  84 +++-
 examples/test_case_data/sandbox/hive-site.xml   |  89 -
 examples/test_case_data/sandbox/mapred-site.xml |  57 ++---
 examples/test_case_data/sandbox/yarn-site.xml   | 127 ---
 12 files changed, 404 insertions(+), 174 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh 
b/dev-support/sync_hbase_cdh_branches.sh
index 472f402..592053a 100755
--- a/dev-support/sync_hbase_cdh_branches.sh
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -37,24 +37,17 @@ trap 'error ${LINENO} ${?}' ERR
 # 
 
 git fetch apache
-git checkout apache/$base-hbase1.x
-git format-patch -1
-git checkout apache/$base-cdh5.7
+git checkout apache/$base-hbase0.98
 git format-patch -1
 
 git checkout apache/$base
 git checkout -b tmp
 git reset apache/$base --hard
 
-git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-mvn clean compile -DskipTests
-git push apache tmp:$base-hbase1.x -f
-rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-
-git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+git am -3 --ignore-whitespace 
0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
 mvn clean compile -DskipTests
-git push apache tmp:$base-cdh5.7 -f
-rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+git push apache tmp:$base-hbase0.98 -f
+rm 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
 
 # clean up
 git checkout master

http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
deleted file mode 100755
index f7780dd..000
--- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-cd ${dir}
-cd ..
-
-mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee 
mvnverify.log

http://git-wip-us.apache.org/repos/asf/kylin/blob/87d10394/dev-support/test_all_against_hdp_2_4_0_0_169.sh
--
diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh 
b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
new file mode 100755
index 000..2a3d24b
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding 

[28/47] kylin git commit: minor, refine cdh profile name

[billyliu]

minor, refine cdh profile name


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db85d667
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db85d667
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db85d667

Branch: refs/heads/KYLIN-2361
Commit: db85d6679c5e1c067cfe2f1c58de0efb88044203
Parents: 87d1039
Author: Hongbin Ma 
Authored: Mon Jan 23 16:14:33 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 16:14:33 2017 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/db85d667/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 520ac0c..bd1882e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1178,7 +1178,7 @@
 
 
 
-cdh-5.7
+cdh5.7
 
 2.6.0-cdh5.7.0
 2.6.0-cdh5.7.0

[34/47] kylin git commit: Add HDFSResourceStore to ResourceStore

[billyliu]

Add HDFSResourceStore to ResourceStore

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c33e38f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c33e38f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c33e38f

Branch: refs/heads/KYLIN-2361
Commit: 4c33e38f3e6c33438ef3bb830875b8ed74ffac5b
Parents: d3276e2
Author: xiefan46 <958034...@qq.com>
Authored: Mon Jan 23 10:51:12 2017 +0800
Committer: shaofengshi 
Committed: Mon Jan 23 16:23:56 2017 +0800

--
 .../kylin/common/persistence/ResourceStore.java   |  5 +
 .../kylin/storage/hdfs/ITHDFSResourceStoreTest.java   | 13 -
 .../apache/kylin/storage/hdfs/ITLockManagerTest.java  | 14 +-
 .../kylin/storage/hbase/HBaseResourceStore.java   |  3 ++-
 .../apache/kylin/storage/hdfs/HDFSResourceStore.java  |  4 +++-
 5 files changed, 23 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 9549569..25a0801 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -75,6 +75,11 @@ abstract public class ResourceStore {
 } catch (Throwable e) {
 logger.warn("Failed to load HBaseResourceStore impl class: " + 
e.toString());
 }
+try {
+
knownImpl.add(ClassUtil.forName("org.apache.kylin.storage.hdfs.HDFSResourceStore",
 ResourceStore.class));
+} catch (Throwable e) {
+logger.warn("Failed to load HDFSResourceStore impl class: " + 
e.toString());
+}
 }
 return knownImpl;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 27d8a3c..47afb01 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -19,17 +19,12 @@
 package org.apache.kylin.storage.hdfs;
 
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.ResourceStoreTest;
 import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-
-/**
- * Created by xiefan on 17-1-10.
- */
 public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
 
 KylinConfig kylinConfig;
@@ -45,13 +40,13 @@ public class ITHDFSResourceStoreTest extends 
HBaseMetadataTestCase {
 this.cleanupTestMetadata();
 }
 
-
 @Test
 public void testResourceStoreBasic() throws Exception {
-KylinConfig config = KylinConfig.getInstanceFromEnv();
-ResourceStore store = new HDFSResourceStore(config);
+String oldUrl = kylinConfig.getMetadataUrl();
+kylinConfig.setProperty("kylin.metadata.url", 
"kylin_default_instance@hdfs");
+HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
 ResourceStoreTest.testAStore(store);
+kylinConfig.setProperty("kylin.metadata.url", oldUrl);
 }
 
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
index 2b58d30..a6cf43b 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
@@ -27,6 +27,8 @@ import org.apache.kylin.common.util.HBaseMetadataTestCase;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -58,6 +60,8 @@ public class ITLockManagerTest extends 

[47/47] kylin git commit: KYLIN-2361 add Tomcat8 ordered class loader

[billyliu]

KYLIN-2361 add Tomcat8 ordered class loader


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a3bd71c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a3bd71c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a3bd71c

Branch: refs/heads/KYLIN-2361
Commit: 9a3bd71c8e5ce9dc13e38560efc556dc862819a1
Parents: 2b60ac6
Author: Billy Liu 
Authored: Sat Feb 4 11:41:49 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 11:42:02 2017 +0800

--
 .../kylin/ext/CustomizedWebappClassloader.java  |   4 +-
 .../kylin/ext/OrderedWebResourceRoot.java   | 286 +++
 .../kylin/ext/WebappOrderedClassLoader.java |  66 +
 3 files changed, 353 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
--
diff --git 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
index f241865..bbf4053 100644
--- 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
+++ 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
@@ -18,14 +18,12 @@
 
 package org.apache.kylin.ext;
 
-import org.apache.catalina.loader.ParallelWebappClassLoader;
-
 /**
  * simple extension to standard ParallelWebappClassLoader
  * the only difference is that CustomizedWebappClassloader is able to delegate 
more packages
  * to parent classloaders
  */
-public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
+public class CustomizedWebappClassloader extends WebappOrderedClassLoader {
 /**
  * Set of package names which are not allowed to be loaded from a webapp
  * class loader without delegating first.

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
--
diff --git 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
new file mode 100644
index 000..9784bd8
--- /dev/null
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.ext;
+
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.catalina.Context;
+import org.apache.catalina.LifecycleException;
+import org.apache.catalina.LifecycleListener;
+import org.apache.catalina.LifecycleState;
+import org.apache.catalina.TrackedWebResource;
+import org.apache.catalina.WebResource;
+import org.apache.catalina.WebResourceRoot;
+import org.apache.catalina.WebResourceSet;
+
+public class OrderedWebResourceRoot implements WebResourceRoot {
+
+private static final String WEB_INF_LIB_PATH = "/WEB-INF/lib";
+
+private static final Comparator WEB_RESOURCE_COMPARATOR = new 
Comparator() {
+@Override
+public int compare(WebResource o1, WebResource o2) {
+return o1.getName().compareTo(o2.getName());
+}
+};
+
+private WebResourceRoot delegate;
+
+public OrderedWebResourceRoot(WebResourceRoot delegate) {
+this.delegate = delegate;
+}
+
+@Override
+public WebResource[] listResources(String path) {
+WebResource[] webResources = delegate.listResources(path);
+
+if (WEB_INF_LIB_PATH.equals(path)) {
+Arrays.sort(webResources, WEB_RESOURCE_COMPARATOR);
+}
+
+return webResources;
+}
+
+@Override
+public void addLifecycleListener(LifecycleListener listener) {
+

[22/47] kylin git commit: KYLIN 2396 GUI for Percentile pre aggregation implementation

[billyliu]

KYLIN 2396 GUI for Percentile pre aggregation implementation

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28673a81
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28673a81
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28673a81

Branch: refs/heads/KYLIN-2361
Commit: 28673a8126ec2ecff8c125c0ecf887ce2565362c
Parents: 6aaf667
Author: chenzhx <346839...@qq.com>
Authored: Fri Jan 20 18:00:20 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 11:32:46 2017 +0800

--
 webapp/app/js/controllers/cubeMeasures.js  | 18 +++
 webapp/app/js/model/cubeConfig.js  |  2 +-
 webapp/app/partials/cubeDesigner/measures.html | 54 -
 3 files changed, 40 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index c8548ce..7c155fe 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -80,6 +80,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   $scope.updateMeasureStatus.editIndex = index;
 }
 $scope.nextParameters = [];
+$scope.measureParamValueColumn=$scope.getCommonMetricColumns();
 $scope.newMeasure = (!!measure)? jQuery.extend(true, 
{},measure):CubeDescModel.createMeasure();
 if(!!measure && measure.function.parameter.next_parameter){
   $scope.nextPara.value = measure.function.parameter.next_parameter.value;
@@ -93,6 +94,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 }else{
   $scope.newMeasure.showDim=false;
 }
+$scope.measureParamValueUpdate();
 if($scope.newMeasure.function.expression=="TOP_N"){
   $scope.convertedColumns=[];
   if($scope.newMeasure.function.configuration==null){
@@ -326,6 +328,19 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   return false;
 }
   }
+
+  $scope.measureParamValueUpdate = function(){
+if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && 
$scope.newMeasure.showDim==true){
+   $scope.measureParamValueColumn=$scope.getAllModelDimMeasureColumns();
+}
+if($scope.newMeasure.function.expression !== 'EXTENDED_COLUMN' && 
$scope.newMeasure.showDim==false){
+   $scope.measureParamValueColumn=$scope.getCommonMetricColumns();
+}
+if($scope.newMeasure.function.expression == 'EXTENDED_COLUMN'){
+  $scope.measureParamValueColumn=$scope.getExtendedHostColumn();
+}
+  }
+
   //map right return type for param
   $scope.measureReturnTypeUpdate = function(){
 
@@ -393,6 +408,9 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
 case "COUNT":
   $scope.newMeasure.function.returntype = "bigint";
   break;
+case "PERCENTILE":
+  $scope.newMeasure.function.returntype = "percentile(100)";
+  break;
 default:
   $scope.newMeasure.function.returntype = "";
   break;

http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/js/model/cubeConfig.js
--
diff --git a/webapp/app/js/model/cubeConfig.js 
b/webapp/app/js/model/cubeConfig.js
index 87413dc..35af1fc 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -20,7 +20,7 @@ KylinApp.constant('cubeConfig', {
 
   //~ Define metadata & class
   measureParamType: ['column', 'constant'],
-  measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 
'RAW','EXTENDED_COLUMN'],
+  measureExpressions: ['SUM', 'MIN', 'MAX', 'COUNT', 'COUNT_DISTINCT',"TOP_N", 
'RAW','EXTENDED_COLUMN','PERCENTILE'],
   dimensionDataTypes: ["string", "tinyint", "int", "bigint", "date"],
   cubePartitionTypes: ['APPEND'],
   joinTypes: [

http://git-wip-us.apache.org/repos/asf/kylin/blob/28673a81/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index 4d2d8ab..4a48843 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -104,7 +104,7 @@
   
   
   
   
@@ -144,35 +144,23 @@
 
 
   
-  

[35/47] kylin git commit: KYLIN-2415 Change back default metadata name to kylin_metadata

[billyliu]

KYLIN-2415 Change back default metadata name to kylin_metadata


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6d247c2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6d247c2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6d247c2

Branch: refs/heads/KYLIN-2361
Commit: b6d247c2c1412d7eb1024048ae7aad0ba5d1ea82
Parents: 971bf80
Author: shaofengshi 
Authored: Tue Jan 24 13:43:04 2017 +0800
Committer: shaofengshi 
Committed: Tue Jan 24 13:43:04 2017 +0800

--
 build/conf/kylin.properties| 2 +-
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
 .../apache/kylin/metadata/realization/IRealizationConstants.java   | 2 +-
 examples/test_case_data/sandbox/kylin.properties   | 2 +-
 .../org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java | 2 +-
 .../java/org/apache/kylin/storage/hbase/HBaseResourceStore.java| 2 +-
 .../org/apache/kylin/storage/hbase/util/StorageCleanupJob.java | 2 +-
 .../main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java | 2 +-
 tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java| 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 196a711..eceb886 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -18,7 +18,7 @@
 ### METADATA | ENV ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_default_instance@hbase
+kylin.metadata.url=kylin_metadata@hbase
 
 # Working folder in HDFS, make sure user has the right access to the hdfs 
directory
 kylin.env.hdfs-working-dir=/kylin

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 75b38ff..07cb124 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -216,7 +216,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 public String getMetadataUrlPrefix() {
 String metadataUrl = getMetadataUrl();
-String defaultPrefix = "kylin_default_instance";
+String defaultPrefix = "kylin_metadata";
 
 if (metadataUrl.endsWith("@hbase")) {
 int cut = metadataUrl.lastIndexOf('@');

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index adc53a1..2a3b01e 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -28,7 +28,7 @@ public class IRealizationConstants {
 
 /**
  * For each cube htable, we leverage htable's metadata to keep track of
- * which kylin server(represented by its kylin_default_instance prefix) 
owns this htable
+ * which kylin server(represented by its kylin_metadata prefix) owns this 
htable
  */
 public final static String HTableTag = "KYLIN_HOST";
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/examples/test_case_data/sandbox/kylin.properties
--
diff --git a/examples/test_case_data/sandbox/kylin.properties 
b/examples/test_case_data/sandbox/kylin.properties
index 2c2da91..fa0c6bc 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -39,7 +39,7 @@ kylin.source.hive.client=cli
 ### STORAGE ###
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_default_instance@hbase
+kylin.metadata.url=kylin_metadata@hbase
 
 
 # The storage for final cube file in hbase

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6d247c2/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
--
diff --git 

[42/47] kylin git commit: KYLIN-2414 minor rename on properties

[billyliu]

KYLIN-2414 minor rename on properties


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3fff6d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3fff6d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3fff6d1

Branch: refs/heads/KYLIN-2361
Commit: c3fff6d19d355e78461fa7f32d02feabc5bf63c8
Parents: e6a9382
Author: shaofengshi 
Authored: Thu Jan 26 10:53:19 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 10:53:19 2017 +0800

--
 build/conf/kylin.properties  |  8 +++-
 .../org/apache/kylin/common/KylinConfigBase.java |  4 ++--
 .../mr/steps/FactDistinctColumnsReducer.java | 19 +--
 3 files changed, 18 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 43ea17d..1232c47 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -127,9 +127,15 @@ kylin.engine.mr.max-reducer-number=500
 
 kylin.engine.mr.mapper-input-rows=100
 
+# Enable dictionary building in MR reducer
+kylin.engine.mr.build-dict-in-reducer=true
+
+# Number of reducers for fetching UHC column distinct values
+kylin.engine.mr.uhc-reducer-count=1
+
 ### CUBE | DICTIONARY ###
 
-# 'auto', 'inmem', 'layer' or 'random' for testing
+# 'auto', 'inmem' or 'layer'
 kylin.cube.algorithm=auto
 
 # A smaller threshold prefers layer, a larger threshold prefers in-mem

http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b25bcc0..6a88fc4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -740,8 +740,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
 }
 
-public boolean isReducerLocalBuildDict() {
-return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", 
"true"));
+public boolean isBuildDictInReducerEnabled() {
+return 
Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer", 
"true"));
 }
 
 public String getYarnStatusCheckUrl() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index cf94b30..5b795c2 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -78,7 +78,7 @@ public class FactDistinctColumnsReducer extends 
KylinReducer 1) {
 int[] uhcIndex = 
CubeManager.getInstance(config).getUHCIndex(cubeDesc);
 int colIndex = reducerIdToColumnIndex.get(taskId);
 if (uhcIndex[colIndex] == 1)
-isReducerLocalBuildDict = false; //for UHC columns, this 
feature should be disabled
+buildDictInReducer = false; //for UHC columns, this 
feature should be disabled
 }
-if (isReducerLocalBuildDict) {
+if (buildDictInReducer) {
 builder = 
DictionaryGenerator.newDictionaryBuilder(col.getType());
 builder.init(null, 0);
 }
-logger.info("Reducer " + taskId + " handling column " + col + ", 
isReducerLocalBuildDict=" + isReducerLocalBuildDict);
+logger.info("Reducer " + taskId + " handling column " + col + ", 
buildDictInReducer=" + buildDictInReducer);
 }
 }
 
@@ -192,7 +191,7 @@ public class FactDistinctColumnsReducer extends 
KylinReducer dict = builder.build();
 outputDict(col, dict);
 }

[21/47] kylin git commit: KYLIN 2396 GUI for Percentile pre aggregation implementation

[billyliu]

KYLIN 2396 GUI for Percentile pre aggregation implementation

Signed-off-by: lidongsjtu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b56bc622
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b56bc622
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b56bc622

Branch: refs/heads/KYLIN-2361
Commit: b56bc622086495f7d0047d01af57efc919b2ecce
Parents: 28673a8
Author: luguosheng <550175...@qq.com>
Authored: Mon Jan 23 11:24:39 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 11:32:46 2017 +0800

--
 webapp/app/js/controllers/cubeMeasures.js  | 2 ++
 webapp/app/partials/cubeDesigner/measures.html | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js 
b/webapp/app/js/controllers/cubeMeasures.js
index 7c155fe..91c38f2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -355,6 +355,8 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, 
$modal,MetaModel,cubes
   $scope.newMeasure.function.parameter.type= 'column';
   $scope.newMeasure.function.returntype = "extendedcolumn(100)";
   return;
+}else if($scope.newMeasure.function.expression=='PERCENTILE'){
+  $scope.newMeasure.function.parameter.type= 'column';
 }else{
   $scope.nextParameterInit();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b56bc622/webapp/app/partials/cubeDesigner/measures.html
--
diff --git a/webapp/app/partials/cubeDesigner/measures.html 
b/webapp/app/partials/cubeDesigner/measures.html
index 4a48843..b7475b6 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -116,7 +116,7 @@
   
   Param Type
   
-  
   
   constant
+ ng-if="newMeasure.function.expression 
== 'COUNT'||newMeasure.function.expression == 'PERCENTILE'"
+ 
>{{newMeasure.function.parameter.type}}
   
   
   

[43/47] kylin git commit: minor, make DefaultSchedulerTest.testDiscard() more stable

[billyliu]

minor, make DefaultSchedulerTest.testDiscard() more stable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0582512e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0582512e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0582512e

Branch: refs/heads/KYLIN-2361
Commit: 0582512ea06bcd1c662903ff69012ba1cc7cc33c
Parents: c3fff6d
Author: Li Yang 
Authored: Thu Jan 26 11:49:04 2017 +0800
Committer: Li Yang 
Committed: Thu Jan 26 11:49:56 2017 +0800

--
 .../apache/kylin/job/SelfStopExecutable.java| 31 
 .../impl/threadpool/DefaultSchedulerTest.java   |  5 ++--
 2 files changed, 27 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java 
b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
index b4f6a98..9a3eb48 100644
--- a/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
+++ b/core-job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
@@ -26,21 +26,40 @@ import org.apache.kylin.job.execution.ExecuteResult;
  */
 public class SelfStopExecutable extends BaseTestExecutable {
 
+volatile boolean doingWork;
+
 public SelfStopExecutable() {
 super();
 }
 
 @Override
 protected ExecuteResult doWork(ExecutableContext context) throws 
ExecuteException {
+doingWork = true;
 try {
-Thread.sleep(5000);
-} catch (InterruptedException e) {
-}
-if (isDiscarded()) {
-return new ExecuteResult(ExecuteResult.State.STOPPED, "stopped");
-} else {
+for (int i = 0; i < 20; i++) {
+sleepOneSecond();
+
+if (isDiscarded())
+return new ExecuteResult(ExecuteResult.State.STOPPED, 
"stopped");
+}
+
 return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
+} finally {
+doingWork = false;
+}
+}
+
+private void sleepOneSecond() {
+try {
+Thread.sleep(1000);
+} catch (InterruptedException e) {
+throw new RuntimeException(e);
 }
 }
 
+public void waitForDoWork() {
+while (doingWork) {
+sleepOneSecond();
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0582512e/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
index bcd6a59..2416311 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
@@ -97,7 +97,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest {
 @Test
 public void testDiscard() throws Exception {
 DefaultChainedExecutable job = new DefaultChainedExecutable();
-BaseTestExecutable task1 = new SelfStopExecutable();
+SelfStopExecutable task1 = new SelfStopExecutable();
 job.addTask(task1);
 jobService.addJob(job);
 waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
@@ -105,8 +105,7 @@ public class DefaultSchedulerTest extends BaseSchedulerTest 
{
 waitForJobFinish(job.getId());
 Assert.assertEquals(ExecutableState.DISCARDED, 
jobService.getOutput(job.getId()).getState());
 Assert.assertEquals(ExecutableState.DISCARDED, 
jobService.getOutput(task1.getId()).getState());
-Thread.sleep(5000);
-System.out.println(job);
+task1.waitForDoWork();
 }
 
 @SuppressWarnings("rawtypes")

[45/47] kylin git commit: minor, stablize DefaultSchedulerTest

[billyliu]

minor, stablize DefaultSchedulerTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a058bfb8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a058bfb8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a058bfb8

Branch: refs/heads/KYLIN-2361
Commit: a058bfb8b0490fe36b7fe4da026028411ed208a5
Parents: 546f88f
Author: Li Yang 
Authored: Fri Feb 3 13:43:56 2017 +0800
Committer: Li Yang 
Committed: Fri Feb 3 13:43:56 2017 +0800

--
 .../job/impl/threadpool/BaseSchedulerTest.java  | 36 ++--
 1 file changed, 26 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a058bfb8/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index fdf5252..1ada9a1 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -31,11 +31,15 @@ import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.job.lock.MockJobLock;
 import org.junit.After;
 import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  */
 public abstract class BaseSchedulerTest extends LocalFileMetadataTestCase {
 
+private static final Logger logger = 
LoggerFactory.getLogger(BaseSchedulerTest.class);
+
 private DefaultScheduler scheduler;
 
 protected ExecutableManager jobService;
@@ -70,19 +74,31 @@ public abstract class BaseSchedulerTest extends 
LocalFileMetadataTestCase {
 }
 
 protected void waitForJobFinish(String jobId) {
-while (true) {
-AbstractExecutable job = jobService.getJob(jobId);
-final ExecutableState status = job.getStatus();
-if (status == ExecutableState.SUCCEED || status == 
ExecutableState.ERROR || status == ExecutableState.STOPPED || status == 
ExecutableState.DISCARDED) {
-break;
-} else {
-try {
-Thread.sleep(5000);
-} catch (InterruptedException e) {
-e.printStackTrace();
+int error = 0;
+final int errorLimit = 3;
+
+while (error < errorLimit) {
+try {
+Thread.sleep(2000);
+} catch (InterruptedException e) {
+e.printStackTrace();
+}
+
+try {
+AbstractExecutable job = jobService.getJob(jobId);
+ExecutableState status = job.getStatus();
+if (status == ExecutableState.SUCCEED || status == 
ExecutableState.ERROR || status == ExecutableState.STOPPED || status == 
ExecutableState.DISCARDED) {
+break;
 }
+} catch (Exception ex) {
+logger.error("", ex);
+error++;
 }
 }
+
+if (error >= errorLimit) {
+throw new RuntimeException("waitForJobFinish() encounters 
exceptions, see logs above");
+}
 }
 
 protected void waitForJobStatus(String jobId, ExecutableState state, long 
interval) {

[09/47] kylin git commit: KYLIN-2411 Kill MR job on pause

[billyliu]

KYLIN-2411 Kill MR job on pause


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b49c9e39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b49c9e39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b49c9e39

Branch: refs/heads/KYLIN-2361
Commit: b49c9e3951c29a21e71f179ee77e2295fbc52ba9
Parents: 38c3e7b
Author: shaofengshi 
Authored: Thu Jan 19 11:02:12 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:37:42 2017 +0800

--
 .../org/apache/kylin/job/execution/ExecutableManager.java| 2 +-
 .../apache/kylin/engine/mr/common/MapReduceExecutable.java   | 8 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 466cdad..48cedb5 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -347,7 +347,7 @@ public class ExecutableManager {
 for (AbstractExecutable task : tasks) {
 if (task.getId().compareTo(stepId) >= 0) {
 logger.debug("rollback task : " + task);
-updateJobOutput(task.getId(), ExecutableState.READY, null, 
null);
+updateJobOutput(task.getId(), ExecutableState.READY, 
Maps.newHashMap(), "");
 }
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b49c9e39/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index f887c4c..6de07ca 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -148,7 +148,7 @@ public class MapReduceExecutable extends AbstractExecutable 
{
 //boolean useKerberosAuth = 
context.getConfig().isGetJobStatusWithKerberos();
 //HadoopStatusChecker statusChecker = new 
HadoopStatusChecker(restStatusCheckUrl, mrJobId, output, useKerberosAuth);
 JobStepStatusEnum status = JobStepStatusEnum.NEW;
-while (!isDiscarded()) {
+while (!isDiscarded() && !isPaused()) {
 
 JobStepStatusEnum newStatus = 
HadoopJobStatusChecker.checkStatus(job, output);
 if (status == JobStepStatusEnum.KILLED) {
@@ -184,7 +184,11 @@ public class MapReduceExecutable extends 
AbstractExecutable {
 }
 }
 
-return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
+if (isDiscarded()) {
+return new ExecuteResult(ExecuteResult.State.DISCARDED, 
output.toString());
+} else {
+return new ExecuteResult(ExecuteResult.State.STOPPED, 
output.toString());
+}
 
 } catch (ReflectiveOperationException e) {
 logger.error("error getMapReduceJobClass, class name:" + 
getParam(KEY_MR_JOB), e);

[37/47] kylin git commit: KYLIN-2242 fix the bug that couldn’t parse partition column max/min value error

[billyliu]

KYLIN-2242 fix the bug that couldn’t parse partition column max/min value 
error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a86569a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a86569a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a86569a5

Branch: refs/heads/KYLIN-2361
Commit: a86569a5c0b49f5a607f642a70af7a826fda3edf
Parents: 571b791
Author: shaofengshi 
Authored: Tue Jan 24 15:59:19 2017 +0800
Committer: shaofengshi 
Committed: Tue Jan 24 15:59:38 2017 +0800

--
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  | 20 +++-
 1 file changed, 15 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a86569a5/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 79fe657..2efd718 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -18,7 +18,9 @@
 
 package org.apache.kylin.engine.mr.steps;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -79,6 +81,10 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 
 private void updateTimeRange(CubeSegment segment) throws IOException {
 final TblColRef partitionCol = 
segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
+
+if (partitionCol == null) {
+return;
+}
 final String factColumnsInputPath = 
this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
 Path colDir = new Path(factColumnsInputPath, 
partitionCol.getIdentity());
 FileSystem fs = HadoopUtil.getWorkingFileSystem();
@@ -88,15 +94,19 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 }
 
 FSDataInputStream is = null;
-long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
+BufferedReader bufferedReader = null;
+InputStreamReader isr = null;
+long minValue, maxValue;
 try {
 is = fs.open(outputFile);
-long min = is.readLong();
-long max = is.readLong();
-minValue = Math.min(min, minValue);
-maxValue = Math.max(max, maxValue);
+isr = new InputStreamReader(is);
+bufferedReader = new BufferedReader(isr);
+minValue = Long.parseLong(bufferedReader.readLine());
+maxValue = Long.parseLong(bufferedReader.readLine());
 } finally {
 IOUtils.closeQuietly(is);
+IOUtils.closeQuietly(isr);
+IOUtils.closeQuietly(bufferedReader);
 }
 logger.info("updateTimeRange step. minValue:" + minValue + " 
maxValue:" + maxValue);
 segment.setDateRangeStart(minValue);

[40/47] kylin git commit: KYLIN-2374 code review

[billyliu]

KYLIN-2374 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eae37ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eae37ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eae37ef

Branch: refs/heads/KYLIN-2361
Commit: 5eae37ef18ca51027c6bb2cfd3410fefc7982f2a
Parents: a2a59c4
Author: shaofengshi 
Authored: Thu Jan 26 09:55:48 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 26 09:55:48 2017 +0800

--
 build/conf/kylin.properties |  3 +-
 build/deploy/spark-defaults.conf|  1 -
 .../apache/kylin/common/KylinConfigBase.java|  8 --
 .../kylin/common/persistence/ResourceStore.java |  3 +
 .../org/apache/kylin/cube/model/CubeDesc.java   |  2 +-
 .../ExtendedColumnMeasureType.java  |  8 +-
 .../storage/hdfs/ITHDFSResourceStoreTest.java   | 36 +++-
 .../kylin/storage/hbase/HBaseResourceStore.java |  3 +-
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 90 +++-
 9 files changed, 97 insertions(+), 57 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index eceb886..43ea17d 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -211,8 +211,9 @@ kylin.engine.spark-conf.spark.executor.memory=4G
 kylin.engine.spark-conf.spark.executor.cores=4
 kylin.engine.spark-conf.spark.executor.instances=8
 kylin.engine.spark-conf.spark.storage.memoryFraction=0.3
-kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.eventLog.enabled=true
 kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
 ## manually upload spark-assembly jar to HDFS and then set this property will 
avoid repeatedly uploading jar at runtime
 
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
 
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/build/deploy/spark-defaults.conf
--
diff --git a/build/deploy/spark-defaults.conf b/build/deploy/spark-defaults.conf
index 36c0ab3..78a4bc9 100644
--- a/build/deploy/spark-defaults.conf
+++ b/build/deploy/spark-defaults.conf
@@ -1,5 +1,4 @@
 spark.yarn.submit.file.replication=1
-spark.eventLog.enabled=true
 spark.yarn.max.executor.failures=3
 spark.driver.extraJavaOptions=-Dhdp.version=current
 spark.yarn.am.extraJavaOptions=-Dhdp.version=current

http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5932197..b1acbbf 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -193,14 +193,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
-public String getRawHdfsWorkingDirectory() {
-String root = getRequired("kylin.env.hdfs-working-dir");
-if (!root.endsWith("/")) {
-root += "/";
-}
-return root;
-}
-
 // 

 // METADATA
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/5eae37ef/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 25a0801..c441618 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -63,6 +63,9 @@ abstract public class ResourceStore {
 public static final String CUBE_STATISTICS_ROOT = "/cube_statistics";
 public 

[01/47] kylin git commit: KYLIN-2348 let choose model freely for sub-queries [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2361 f577a7da0 -> 9a3bd71c8 (forced update)


KYLIN-2348 let choose model freely for sub-queries


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d52bb8bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d52bb8bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d52bb8bc

Branch: refs/heads/KYLIN-2361
Commit: d52bb8bc05bd530362fe0970b8e8a85f8b2289bc
Parents: 51c6571
Author: Li Yang 
Authored: Wed Jan 18 22:46:24 2017 +0800
Committer: Li Yang 
Committed: Wed Jan 18 22:46:24 2017 +0800

--
 .../apache/kylin/query/relnode/OLAPTableScan.java   |  2 ++
 .../apache/kylin/query/routing/ModelChooser.java| 16 
 2 files changed, 10 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d52bb8bc/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 452170a..f7877be 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -53,6 +53,7 @@ import org.apache.calcite.rel.rules.JoinPushExpressionsRule;
 import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
 import org.apache.calcite.rel.rules.JoinUnionTransposeRule;
 import org.apache.calcite.rel.rules.ReduceExpressionsRule;
+import org.apache.calcite.rel.rules.SemiJoinRule;
 import org.apache.calcite.rel.rules.SortJoinTransposeRule;
 import org.apache.calcite.rel.rules.SortUnionTransposeRule;
 import org.apache.calcite.rel.type.RelDataType;
@@ -174,6 +175,7 @@ public class OLAPTableScan extends TableScan implements 
OLAPRel, EnumerableRel {
 planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
 planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
 planner.removeRule(DateRangeRules.FILTER_INSTANCE);
+planner.removeRule(SemiJoinRule.INSTANCE);
 // distinct count will be split into a separated query that is joined 
with the left query
 planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d52bb8bc/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java 
b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index 2517dc5..f979f7b 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -53,17 +53,17 @@ public class ModelChooser {
 IdentityHashMap candidates = new 
IdentityHashMap<>();
 
 // attempt one model for all contexts
-Set reals = attemptSelectModel(contexts);
-if (reals != null) {
-for (OLAPContext ctx : contexts) {
-candidates.put(ctx, reals);
-}
-return candidates;
-}
+//Set reals = attemptSelectModel(contexts);
+//if (reals != null) {
+//for (OLAPContext ctx : contexts) {
+//candidates.put(ctx, reals);
+//}
+//return candidates;
+//}
 
 // try different model for different context
 for (OLAPContext ctx : contexts) {
-reals = attemptSelectModel(ImmutableList.of(ctx));
+Set reals = 
attemptSelectModel(ImmutableList.of(ctx));
 if (reals == null)
 throw new NoRealizationFoundException("No model found for" + 
toErrorMsg(ctx));
 

[06/47] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[billyliu]

KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dfb5fac9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dfb5fac9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dfb5fac9

Branch: refs/heads/KYLIN-2361
Commit: dfb5fac9f0d9c13f1128206b73e683b515fe3d72
Parents: 61833d9
Author: Billy Liu 
Authored: Thu Jan 19 14:54:20 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 19 14:54:20 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3941 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2430 insertions(+), 1571 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b916df2..1b327fe 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -36,7 +36,16 @@
 
 org.apache.calcite
 calcite-core
+
+
+org.apache.calcite.avatica
+avatica-core
+
+
+
+
+org.apache.calcite.avatica
+avatica
 
 
-
 

[02/47] kylin git commit: KYLIN-2410 let global dictionary use hadoop conf in mapper/reducer context

[billyliu]

KYLIN-2410 let global dictionary use hadoop conf in mapper/reducer context


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eddb695b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eddb695b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eddb695b

Branch: refs/heads/KYLIN-2361
Commit: eddb695ba13de282ada123f358aff723cb414316
Parents: d52bb8b
Author: Yang Li 
Authored: Thu Jan 19 08:56:22 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 19 08:56:22 2017 +0800

--
 core-common/pom.xml |   5 +
 .../apache/kylin/common/util/HadoopUtil.java| 146 ++
 .../apache/kylin/dict/AppendTrieDictionary.java |  13 +-
 .../kylin/dict/AppendTrieDictionaryChecker.java |  13 +-
 .../org/apache/kylin/dict/CachedTreeMap.java|   7 +-
 .../kylin/dict/AppendTrieDictionaryTest.java|   6 +-
 .../apache/kylin/dict/CachedTreeMapTest.java|   8 +-
 .../apache/kylin/engine/mr/DFSFileTable.java|   1 +
 .../kylin/engine/mr/DFSFileTableReader.java |   1 +
 .../org/apache/kylin/engine/mr/HadoopUtil.java  | 150 ---
 .../org/apache/kylin/engine/mr/KylinMapper.java |   1 +
 .../apache/kylin/engine/mr/KylinReducer.java|   1 +
 .../kylin/engine/mr/SortedColumnDFSFile.java|   1 +
 .../engine/mr/common/AbstractHadoopJob.java |   2 +-
 .../kylin/engine/mr/common/CubeStatsReader.java |   2 +-
 .../engine/mr/common/MapReduceExecutable.java   |   2 +-
 .../engine/mr/steps/CreateDictionaryJob.java|   2 +-
 .../mr/steps/FactDistinctColumnsReducer.java|   2 +-
 .../kylin/engine/mr/steps/InMemCuboidJob.java   |   2 +-
 .../engine/mr/steps/MergeStatisticsStep.java|   2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  |   2 +-
 .../engine/mr/steps/SaveStatisticsStep.java |   2 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |   2 +-
 .../steps/FactDistinctColumnsReducerTest.java   |   2 +-
 .../HiveToBaseCuboidMapperPerformanceTest.java  |   2 +-
 .../engine/mr/steps/MergeCuboidJobTest.java |   2 +-
 .../kylin/engine/mr/steps/NDCuboidJobTest.java  |   2 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |   2 +-
 .../kylin/cube/ITDictionaryManagerTest.java |   2 +-
 .../kylin/provision/BuildCubeWithEngine.java|   2 +-
 .../apache/kylin/query/ITMassInQueryTest.java   |   2 +-
 .../kylin/storage/hbase/ITHdfsOpsTest.java  |   2 +-
 .../rest/controller/StreamingController.java|   2 +-
 .../apache/kylin/rest/service/ModelService.java |   2 +-
 .../apache/kylin/rest/service/TableService.java |   2 +-
 .../apache/kylin/source/hive/HiveMRInput.java   |   2 +-
 .../source/hive/HiveSourceTableLoader.java  |   2 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |   2 +-
 .../kylin/storage/hbase/HBaseConnection.java|   2 +-
 .../kylin/storage/hbase/HBaseResourceStore.java |   2 +-
 .../storage/hbase/steps/DeprecatedGCStep.java   |   2 +-
 .../steps/HDFSPathGarbageCollectionStep.java|   2 +-
 .../hbase/steps/SequenceFileCuboidWriter.java   |   2 +-
 .../storage/hbase/util/CubeMigrationCLI.java|   2 +-
 .../storage/hbase/util/StorageCleanupJob.java   |   2 +-
 .../storage/hbase/steps/CreateHTableTest.java   |   2 +-
 .../hbase/steps/CubeHFileMapper2Test.java   |   2 +-
 .../steps/RangeKeyDistributionJobTest.java  |   2 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java |   2 +-
 .../apache/kylin/tool/MrJobInfoExtractor.java   |   2 +-
 .../apache/kylin/tool/StorageCleanupJob.java|   2 +-
 .../org/apache/kylin/tool/util/ToolUtil.java|   2 +-
 52 files changed, 216 insertions(+), 215 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 25b10a7..016d470 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -78,6 +78,11 @@
 
 
 
+org.apache.hadoop
+hadoop-common
+provided
+
+
 junit
 junit
 test

http://git-wip-us.apache.org/repos/asf/kylin/blob/eddb695b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
new file mode 100644
index 000..390c209
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work 

[05/47] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index e0782ce..171e299 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -16,24 +16,6 @@
  */
 package org.apache.calcite.sql2rel;
 
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
 import org.apache.calcite.avatica.util.Spaces;
 import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.plan.Convention;
@@ -85,7 +67,6 @@ import org.apache.calcite.rel.stream.Delta;
 import org.apache.calcite.rel.stream.LogicalDelta;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -116,6 +97,7 @@ import org.apache.calcite.sql.SqlCallBinding;
 import org.apache.calcite.sql.SqlDataTypeSpec;
 import org.apache.calcite.sql.SqlDelete;
 import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainFormat;
 import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.calcite.sql.SqlFunction;
 import org.apache.calcite.sql.SqlIdentifier;
@@ -177,17 +159,37 @@ import org.apache.calcite.util.NumberUtil;
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Util;
 import org.apache.calcite.util.trace.CalciteTrace;
-import org.slf4j.Logger;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+
+import org.slf4j.Logger;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
 
 /*
  * The code has synced with calcite. Hope one day, we could remove the 
hardcode override point.
@@ -209,42 +211,56 @@ import com.google.common.collect.Sets;
 public class SqlToRelConverter {
 //~ Static fields/initializers 
-
 
+protected static final Logger SQL2REL_LOGGER =
+CalciteTrace.getSqlToRelTracer();
+
+private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
 /** Size of the smallest IN list that will be converted to a semijoin to a
  * static table. */
-public static final int DEFAULT_IN_SUBQUERY_THRESHOLD = 20;
-protected static final Logger SQL2REL_LOGGER = 
CalciteTrace.getSqlToRelTracer();
-private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+
+@Deprecated // to be removed before 2.0
+public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
+DEFAULT_IN_SUB_QUERY_THRESHOLD;
 
 //~ Instance fields 

-public final SqlToRelConverter.Config config;
-public final RelOptTable.ViewExpander viewExpander;
+
 protected final SqlValidator validator;
 protected final RexBuilder rexBuilder;
 protected final Prepare.CatalogReader catalogReader;
 protected final RelOptCluster cluster;
+private DefaultValueFactory defaultValueFactory;
+private SubQueryConverter subQueryConverter;
 protected final List leaves = new ArrayList<>();
-protected final RelDataTypeFactory typeFactory;
 private final List dynamicParamSqlNodes = new 
ArrayList<>();
 

[03/47] kylin git commit: KYLIN-2410 remove force qualified path in HadoopUtil.getFileSystem

[billyliu]

KYLIN-2410 remove force qualified path in HadoopUtil.getFileSystem


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61833d95
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61833d95
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61833d95

Branch: refs/heads/KYLIN-2361
Commit: 61833d952daa3ed7e3938f6e310caca5a52a4974
Parents: eddb695
Author: Yang Li 
Authored: Thu Jan 19 10:35:50 2017 +0800
Committer: Yang Li 
Committed: Thu Jan 19 12:42:44 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/util/HadoopUtil.java| 3 ---
 1 file changed, 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/61833d95/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 390c209..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -82,9 +82,6 @@ public class HadoopUtil {
 }
 
 public static FileSystem getFileSystem(Path path, Configuration conf) {
-if (StringUtils.isBlank(path.toUri().getScheme()))
-throw new IllegalArgumentException("Path must be qualified: " + 
path);
-
 try {
 return path.getFileSystem(conf);
 } catch (IOException e) {

[38/47] kylin git commit: KYLIN-2374 fix path bug in HDFSResourceStore

[billyliu]

KYLIN-2374 fix path bug in HDFSResourceStore

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56293411
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56293411
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56293411

Branch: refs/heads/KYLIN-2361
Commit: 5629341122abe18d9576f71a0ea99ac83c327c21
Parents: a86569a
Author: xiefan46 <958034...@qq.com>
Authored: Wed Jan 25 10:41:40 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Jan 25 11:19:13 2017 +0800

--
 .../kylin/storage/hdfs/HDFSResourceStore.java   | 23 +++-
 1 file changed, 13 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/56293411/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index 0ad6964..38acfb0 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -18,7 +18,15 @@
 
 package org.apache.kylin.storage.hdfs;
 
-import com.google.common.collect.Lists;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -32,14 +40,7 @@ import org.apache.kylin.common.util.HadoopUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Lists;
 
 public class HDFSResourceStore extends ResourceStore {
 
@@ -63,7 +64,7 @@ public class HDFSResourceStore extends ResourceStore {
 int cut = metadataUrl.indexOf('@');
 String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : 
metadataUrl.substring(0, cut);
 String hdfsUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 
1);
-if(!hdfsUrl.equals("hdfs"))
+if (!hdfsUrl.equals("hdfs"))
 throw new IOException("Can not create HDFSResourceStore. Url not 
match. Url:" + hdfsUrl);
 metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME;
 logger.info("meta dir name :" + metaDirName);
@@ -245,6 +246,8 @@ public class HDFSResourceStore extends ResourceStore {
 }
 
 private Path getRealHDFSPath(String resourcePath) {
+if (resourcePath.equals("/"))
+return this.hdfsMetaPath;
 if (resourcePath.startsWith("/") && resourcePath.length() > 1)
 resourcePath = resourcePath.substring(1, resourcePath.length());
 return new Path(this.hdfsMetaPath, resourcePath);

[17/47] kylin git commit: Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs"

[billyliu]

Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs"

This reverts commit 7de8aa1203a72bad105ed692f7100535939b03af.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7576a09f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7576a09f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7576a09f

Branch: refs/heads/KYLIN-2361
Commit: 7576a09f29b75f2eb80a1b5ca9c65fe5c81b1144
Parents: 7de8aa1
Author: shaofengshi 
Authored: Sun Jan 22 11:38:17 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 11:38:17 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 --
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 +--
 .../mr/steps/FactDistinctColumnsReducer.java| 240 ---
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 187 insertions(+), 175 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,10 +26,8 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -142,18 +140,4 @@ public class HadoopUtil {
 }
 }
 
-public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
-FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
-@Override
-public boolean accept(Path path) {
-return path.getName().startsWith(filter);
-}
-});
-
-if (fileStatus.length == 1) {
-return fileStatus[0].getPath();
-} else {
-return null;
-}
-}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index c34a904..696b22a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
+return getRealizationRootPath(jobId) + "/statistics";
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 602b4bb..0281539 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,16 +53,9 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
+String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
-String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
-
-String CFG_OUTPUT_COLUMN = 

[13/47] kylin git commit: KYLIN 2370 Refine unload and reload table

[billyliu]

KYLIN 2370 Refine unload and reload table

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a853a7c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a853a7c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a853a7c4

Branch: refs/heads/KYLIN-2361
Commit: a853a7c4b6b7d42cc3ffd066b618f5ea72ca0d9f
Parents: 85a1eb3
Author: chenzhx <346839...@qq.com>
Authored: Mon Jan 9 15:09:16 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 16:21:11 2017 +0800

--
 webapp/app/css/AdminLTE.css |  27 +++
 webapp/app/js/controllers/sourceMeta.js | 175 ---
 .../js/directives/kylin_abn_tree_directive.js   |   7 +-
 .../app/partials/tables/source_table_tree.html  |   6 +-
 webapp/app/partials/tables/table_unload.html|  33 
 5 files changed, 111 insertions(+), 137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/css/AdminLTE.css
--
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index 857dbf7..c7740d1 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -4800,3 +4800,30 @@ Gradient Background colors
 white-space: normal !important;
   }
 }
+
+.abn-tree .abn-tree-row .tree-table-btn{
+width: 4%;
+float: right;
+border-radius: 5px;
+margin-right: 5px;
+padding: 3px 16px 2px 5px;
+color: #ff;
+position:static;
+}
+.abn-tree .abn-tree-row .tree-table-btn .tooltip{
+overflow:visible;
+}
+.abn-tree .abn-tree-row .btn-info:hover{
+   background-color:#269abc;
+}
+.abn-tree > .abn-tree-row.active > .btn-info{
+background-color:#269abc;
+border-left-color:#269abc;
+}
+.abn-tree .abn-tree-row .btn-success:hover{
+background-color:#008d4c;
+}
+.abn-tree > .abn-tree-row.active > .btn-success{
+background-color:#008d4c;
+border-left-color:#008d4c;
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/js/controllers/sourceMeta.js
--
diff --git a/webapp/app/js/controllers/sourceMeta.js 
b/webapp/app/js/controllers/sourceMeta.js
index a53a35f..5421673 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -132,31 +132,83 @@ KylinApp
   });
 };
 
-$scope.openUnLoadModal = function () {
-  if(!$scope.projectModel.selectedProject){
-SweetAlert.swal('Oops...', "Please select a project.", 'info');
+$scope.reloadTable = function (tableName,projectName){
+  loadingRequest.show();
+  TableService.loadHiveTable({tableName: tableName, action: projectName}, 
{calculate: $scope.isCalculate}, function (result) {
+var loadTableInfo = "";
+angular.forEach(result['result.loaded'], function (table) {
+  loadTableInfo += "\n" + table;
+})
+var unloadedTableInfo = "";
+angular.forEach(result['result.unloaded'], function (table) {
+  unloadedTableInfo += "\n" + table;
+})
+if (result['result.unloaded'].length != 0 && 
result['result.loaded'].length == 0) {
+  SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + 
unloadedTableInfo, 'error');
+}
+if (result['result.loaded'].length != 0 && 
result['result.unloaded'].length == 0) {
+  SweetAlert.swal('Success!', 'The following table(s) have been 
successfully loaded: ' + loadTableInfo, 'success');
+}
+if (result['result.loaded'].length != 0 && 
result['result.unloaded'].length != 0) {
+  SweetAlert.swal('Partial loaded!', 'The following table(s) have been 
successfully loaded: ' + loadTableInfo + "\n\n Failed to load following 
table(s):" + unloadedTableInfo, 'warning');
+}
+loadingRequest.hide();
+$scope.aceSrcTbLoaded(true);
+  }, function (e) {
+if (e.data && e.data.exception) {
+  var message = e.data.exception;
+  var msg = !!(message) ? message : 'Failed to take action.';
+  SweetAlert.swal('Oops...', msg, 'error');
+} else {
+  SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+}
+loadingRequest.hide();
+  })
+}
+
+
+
+$scope.removeList = function (tableName,projectName) {
+  if (tableName.trim() === "") {
+SweetAlert.swal('', 'Please input table(s) you want to unload.', 
'info');
 return;
   }
-  $modal.open({
-templateUrl: 'removeHiveTable.html',
-controller: ModalInstanceCtrl,
-backdrop : 'static',
-resolve: {
-  tableNames: function () {
-return $scope.tableNames;

[19/47] kylin git commit: Merge branch 'KYLIN-2394'

[billyliu]

Merge branch 'KYLIN-2394'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c4591a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c4591a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c4591a6

Branch: refs/heads/KYLIN-2361
Commit: 2c4591a609c8d8141c880722a3b8b282b785133e
Parents: d264339 d34f2d3
Author: Billy Liu 
Authored: Sun Jan 22 17:43:16 2017 +0800
Committer: Billy Liu 
Committed: Sun Jan 22 17:43:16 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3782 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2348 insertions(+), 1494 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2c4591a6/kylin-it/pom.xml
--

[20/47] kylin git commit: minor, add config kylin.cube.allow-appear-in-multiple-projects

[billyliu]

minor, add config kylin.cube.allow-appear-in-multiple-projects


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6aaf6676
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6aaf6676
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6aaf6676

Branch: refs/heads/KYLIN-2361
Commit: 6aaf6676b5e904260f16d1222bfe99013a095963
Parents: 2c4591a
Author: Hongbin Ma 
Authored: Mon Jan 23 11:31:38 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Jan 23 11:31:38 2017 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java  | 9 ++---
 .../src/main/java/org/apache/kylin/engine/mr/CubingJob.java | 7 ++-
 2 files changed, 12 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6aaf6676/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 74903d5..05df177 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -181,7 +181,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 if (!root.endsWith("/")) {
 root += "/";
 }
-
+
 // make sure path qualified
 if (!root.contains("://")) {
 if (!root.startsWith("/"))
@@ -189,7 +189,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 else
 root = "hdfs://" + root;
 }
-
+
 return new 
StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', 
'-')).append("/").toString();
 }
 
@@ -320,6 +320,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
 }
 
+public boolean allowCubeAppearInMultipleProjects() {
+return 
Boolean.parseBoolean(getOptional("kylin.cube.allow-appear-in-multiple-projects",
 "false"));
+}
+
 // 

 // JOB
 // 

@@ -767,7 +771,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "10.0"));
 }
 
-
 public int getSparkMinPartition() {
 return Integer.valueOf(getOptional("kylin.engine.spark.min-partition", 
"1"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6aaf6676/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 64e9edb..5aa7d72 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -84,7 +84,12 @@ public class CubingJob extends DefaultChainedExecutable {
 if (projList == null || projList.size() == 0) {
 throw new RuntimeException("Cannot find the project containing the 
cube " + cube.getName() + "!!!");
 } else if (projList.size() >= 2) {
-throw new RuntimeException("Find more than one project containing 
the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! 
");
+String msg = "Find more than one project containing the cube " + 
cube.getName() + ". It does't meet the uniqueness requirement!!! ";
+if (!config.getConfig().allowCubeAppearInMultipleProjects()) {
+throw new RuntimeException(msg);
+} else {
+logger.warn(msg);
+}
 }
 
 CubingJob result = new CubingJob();

[26/47] kylin git commit: KYLIN-2413 a new profile for cdh binary packages

[billyliu]

KYLIN-2413 a new profile for cdh binary packages


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb73da7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb73da7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb73da7a

Branch: refs/heads/KYLIN-2361
Commit: eb73da7a0b27540af54fec23b561b390405620c5
Parents: 169c3de
Author: lidongsjtu 
Authored: Mon Jan 23 11:30:13 2017 +0800
Committer: lidongsjtu 
Committed: Mon Jan 23 13:03:29 2017 +0800

--
 build/script/build.sh   |   2 +-
 build/script/package.sh |  11 +-
 .../kylin/engine/mr/steps/MockupMapContext.java |  11 +-
 pom.xml | 114 +++
 .../storage/hbase/steps/MockupMapContext.java   |  17 +--
 tool/pom.xml|  10 ++
 6 files changed, 145 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/build/script/build.sh
--
diff --git a/build/script/build.sh b/build/script/build.sh
index c68ca94..458c612 100755
--- a/build/script/build.sh
+++ b/build/script/build.sh
@@ -22,7 +22,7 @@ echo "package back-end"
 dir=$(dirname ${0})
 cd ${dir}/../..
 
-mvn clean install -DskipTests   || { exit 1; }
+mvn clean install -DskipTests $@ || { exit 1; }
 
 #package webapp
 echo 'package front-end'

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/build/script/package.sh
--
diff --git a/build/script/package.sh b/build/script/package.sh
index c850ec3..a58a5ce 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -74,9 +74,8 @@ cat << EOF > build/commit_SHA1
 EOF
 git rev-parse HEAD >> build/commit_SHA1
 
-sh build/script/build.sh || { exit 1; }
-sh build/script/download-tomcat.sh || { exit 1; }
-sh build/script/download-spark.sh || { exit 1; }
-sh build/script/prepare.sh || { exit 1; }
-sh build/script/compress.sh || { exit 1; }
-
+sh build/script/build.sh $@ || { exit 1; }
+sh build/script/download-tomcat.sh  || { exit 1; }
+sh build/script/download-spark.sh   || { exit 1; }
+sh build/script/prepare.sh  || { exit 1; }
+sh build/script/compress.sh || { exit 1; }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index dfbba14..92115c9 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -47,7 +47,7 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
  * 
  */
 @SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext implements MapContext{
+public class MockupMapContext implements MapContext {
 
 private Configuration hconf;
 
@@ -60,7 +60,7 @@ public class MockupMapContext implements MapContext{
 return new WrappedMapper().getMapContext(new MockupMapContext(hconf, 
outKV));
 }
 
-public MockupMapContext(Configuration hconf, Object[] outKV){
+public MockupMapContext(Configuration hconf, Object[] outKV) {
 this.hconf = hconf;
 this.outKV = outKV;
 }
@@ -109,7 +109,6 @@ public class MockupMapContext implements MapContext{
 throw new NotImplementedException();
 }
 
-@Override
 public float getProgress() {
 throw new NotImplementedException();
 }
@@ -224,7 +223,6 @@ public class MockupMapContext implements MapContext{
 throw new NotImplementedException();
 }
 
-@Override
 public boolean getTaskCleanupNeeded() {
 throw new NotImplementedException();
 }
@@ -239,7 +237,6 @@ public class MockupMapContext implements MapContext{
 throw new NotImplementedException();
 }
 
-@Override
 public IntegerRanges getProfileTaskRange(boolean isMap) {
 throw new NotImplementedException();
 }
@@ -318,4 +315,8 @@ public class MockupMapContext implements MapContext{
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+public boolean userClassesTakesPrecedence() {
+throw new NotImplementedException();
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eb73da7a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 

[11/47] kylin git commit: minor, fix false positive warning in test output

[billyliu]

minor, fix false positive warning in test output


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85a1eb39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85a1eb39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85a1eb39

Branch: refs/heads/KYLIN-2361
Commit: 85a1eb399d8dcd099d4607aa802faa5ccdf4d2bf
Parents: 1e4ae54
Author: Billy Liu 
Authored: Fri Jan 20 08:13:11 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 08:13:11 2017 +0800

--
 .../org/apache/kylin/common/util/AbstractKylinTestCase.java   | 7 ++-
 .../java/org/apache/kylin/dict/lookup/SnapshotManager.java| 6 ++
 .../src/main/java/org/apache/kylin/job/dao/ExecutableDao.java | 6 ++
 .../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 2 +-
 examples/test_case_data/localmeta/cube_desc/ssb.json  | 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube1.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube2.json| 4 ++--
 examples/test_case_data/localmeta/cube_desc/ssb_cube3.json| 4 ++--
 .../cube_desc/test_kylin_cube_without_slr_left_join_desc.json | 4 ++--
 .../localmeta/cube_desc/test_streaming_table_cube_desc.json   | 2 +-
 examples/test_case_data/localmeta/kylin.properties| 2 +-
 examples/test_case_data/sandbox/kylin.properties  | 2 +-
 kylin-it/pom.xml  | 2 +-
 13 files changed, 33 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 2154c32..4a9804d 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -31,12 +31,17 @@ public abstract class AbstractKylinTestCase {
 public static final String[] SERVICES_WITH_CACHE = { //
 "org.apache.kylin.cube.CubeManager", //
 "org.apache.kylin.cube.CubeDescManager", //
+"org.apache.kylin.dict.lookup.SnapshotManager", //
+"org.apache.kylin.dict.DictionaryManager", //
 "org.apache.kylin.storage.hybrid.HybridManager", //
 "org.apache.kylin.metadata.realization.RealizationRegistry", //
 "org.apache.kylin.metadata.project.ProjectManager", //
 "org.apache.kylin.metadata.MetadataManager", //
+"org.apache.kylin.metadata.cachesync.Broadcaster", //
+"org.apache.kylin.metadata.badquery.BadQueryHistoryManager", //
 "org.apache.kylin.job.impl.threadpool.DistributedScheduler", //
-"org.apache.kylin.job.manager.ExecutableManager", //
+"org.apache.kylin.job.execution.ExecutableManager", //
+"org.apache.kylin.job.dao.ExecutableDao" //
 };
 
 public abstract void createTestMetadata() throws Exception;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index 085158a..b45d017 100644
--- 
a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ 
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -66,6 +66,12 @@ public class SnapshotManager {
 return r;
 }
 
+public static void clearCache() {
+synchronized (SERVICE_CACHE) {
+SERVICE_CACHE.clear();
+}
+}
+
 // 

 
 private KylinConfig config;

http://git-wip-us.apache.org/repos/asf/kylin/blob/85a1eb39/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java 
b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index 5cae5ac..96505e6 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -64,6 +64,12 @@ public class ExecutableDao {
 return r;
 }
 
+public static void clearCache() {
+

[10/47] kylin git commit: KYLIN-2375 change default ehcache size to 256M

[billyliu]

KYLIN-2375 change default ehcache size to 256M


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e4ae540
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e4ae540
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e4ae540

Branch: refs/heads/KYLIN-2361
Commit: 1e4ae540bf194c7e67bb622c2eb0cd0ff1a6657c
Parents: 2196975
Author: shaofengshi 
Authored: Thu Jan 19 17:44:19 2017 +0800
Committer: shaofengshi 
Committed: Thu Jan 19 17:44:19 2017 +0800

--
 server/src/main/resources/ehcache-test.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e4ae540/server/src/main/resources/ehcache-test.xml
--
diff --git a/server/src/main/resources/ehcache-test.xml 
b/server/src/main/resources/ehcache-test.xml
index eb39774..bffe27a 100644
--- a/server/src/main/resources/ehcache-test.xml
+++ b/server/src/main/resources/ehcache-test.xml
@@ -12,7 +12,7 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
->
+>
 

[1/2] kylin git commit: minor update

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/dep-clean b75223197 -> b88e5586b


minor update


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94b60d27
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94b60d27
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94b60d27

Branch: refs/heads/dep-clean
Commit: 94b60d27ef4a823e1b6c6dcffc9d4aa620e1da63
Parents: b752231
Author: Billy Liu 
Authored: Mon Feb 6 14:35:19 2017 +0800
Committer: Billy Liu 
Committed: Mon Feb 6 14:35:19 2017 +0800

--
 jdbc/pom.xml |  2 +-
 source-kafka/pom.xml | 14 ++
 2 files changed, 3 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/94b60d27/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index e22a180..34bdc8a 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -70,8 +70,8 @@
 
 
 com.google.protobuf:*
-commons-codec:*
 commons-logging:*
+org.slf4j:jul-to-slf4j
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/94b60d27/source-kafka/pom.xml
--
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index b8fdf46..da5e033 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -35,17 +35,11 @@
 
 
 org.apache.kylin
-kylin-engine-mr
-
-
-
-org.apache.kylin
 kylin-core-common
 
-
 
-org.apache.commons
-commons-lang3
+org.apache.kylin
+kylin-engine-mr
 
 
 
@@ -78,9 +72,5 @@
 test-jar
 test
 
-
-
-
 
-
 
\ No newline at end of file

[2/2] kylin git commit: update kyro-shaded as provided

[billyliu]

update kyro-shaded as provided


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b88e5586
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b88e5586
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b88e5586

Branch: refs/heads/dep-clean
Commit: b88e5586b34117d45d42ba46d61b8bdee4760e3e
Parents: 94b60d2
Author: Billy Liu 
Authored: Mon Feb 6 18:40:13 2017 +0800
Committer: Billy Liu 
Committed: Mon Feb 6 18:40:13 2017 +0800

--
 core-cube/pom.xml | 8 
 pom.xml   | 5 +
 server/pom.xml| 4 
 3 files changed, 13 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b88e5586/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 5bcc112..fe80783 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -48,10 +48,6 @@
 commons-collections
 commons-collections
 
-
-com.esotericsoftware
-kryo-shaded
-
 
 
 
@@ -70,6 +66,10 @@
 com.google.code.findbugs
 jsr305
 
+
+com.esotericsoftware
+kryo-shaded
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b88e5586/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d8e552..c717fea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -550,26 +550,31 @@
 org.apache.spark
 spark-core_2.10
 ${spark.version}
+provided
 
 
 org.apache.spark
 spark-sql_2.10
 ${spark.version}
+provided
 
 
 org.apache.spark
 spark-hive_2.10
 ${spark.version}
+provided
 
 
 com.esotericsoftware
 kryo-shaded
 ${kryo.version}
+provided
 
 
 org.reflections
 reflections
 ${reflections.version}
+provided
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b88e5586/server/pom.xml
--
diff --git a/server/pom.xml b/server/pom.xml
index 454ed6b..3c96e60 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -233,6 +233,10 @@
 jetty-all
 org.eclipse.jetty.aggregate
 
+
+calcite-core
+org.apache.calcite
+
 
 
 

kylin git commit: upgrade jackson and update it to compile scope

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/dep-clean 778d05a75 -> b75223197


upgrade jackson and update it to compile scope


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b7522319
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b7522319
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b7522319

Branch: refs/heads/dep-clean
Commit: b75223197283427cf504a0ebddf4fe19ecafe655
Parents: 778d05a
Author: Billy Liu 
Authored: Sun Feb 5 10:09:07 2017 +0800
Committer: Billy Liu 
Committed: Sun Feb 5 10:09:07 2017 +0800

--
 core-common/pom.xml |  4 ++--
 core-cube/pom.xml   |  4 
 core-dictionary/pom.xml |  6 --
 core-job/pom.xml|  4 
 core-metadata/pom.xml   |  4 
 core-storage/pom.xml|  4 
 engine-mr/pom.xml   |  5 -
 jdbc/pom.xml|  2 ++
 pom.xml | 23 +++
 source-kafka/pom.xml|  4 
 storage-hbase/pom.xml   |  5 -
 tool/pom.xml|  6 --
 12 files changed, 15 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b7522319/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 316e316..95d3c29 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -34,7 +34,6 @@
 
 
 
-
 
 org.apache.commons
 commons-lang3
@@ -47,11 +46,12 @@
 org.apache.httpcomponents
 httpclient
 
-
 
 com.fasterxml.jackson.core
 jackson-databind
 
+
+
 
 com.google.code.findbugs
 jsr305

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7522319/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index ab8d896..5bcc112 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -63,10 +63,6 @@
 commons-io
 
 
-com.fasterxml.jackson.core
-jackson-databind
-
-
 com.google.guava
 guava
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7522319/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index fe3910b..5d839dd 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -39,12 +39,6 @@
 kylin-core-metadata
 
 
-
-
-com.fasterxml.jackson.core
-jackson-databind
-
-
 
 
 org.apache.hadoop

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7522319/core-job/pom.xml
--
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 5893b75..9478555 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -49,10 +49,6 @@
 commons-io
 
 
-com.fasterxml.jackson.core
-jackson-databind
-
-
 org.apache.curator
 curator-recipes
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7522319/core-metadata/pom.xml
--
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index b5f6394..0832e80 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -53,10 +53,6 @@
 commons-io
 
 
-com.fasterxml.jackson.core
-jackson-databind
-
-
 com.google.guava
 guava
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7522319/core-storage/pom.xml
--
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index e6dd7e6..8c3da5f 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -41,10 +41,6 @@
 
 
 
-com.fasterxml.jackson.core
-jackson-databind
-
-
 com.google.guava
 guava
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b7522319/engine-mr/pom.xml
--
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 0c39e31..b4ee16e 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -111,11 +111,6 @@
 junit
 test
 
-
-com.fasterxml.jackson.core
-jackson-databind
-test
-
 
 
 

[kylin] Git Push Summary

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/dep-clean [deleted] b88e5586b

[2/2] kylin git commit: KYLIN-2428 refine JDBC shade

[billyliu]

KYLIN-2428 refine JDBC shade


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/96a41592
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/96a41592
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/96a41592

Branch: refs/heads/KYLIN-2428
Commit: 96a4159274f6896a0cdd7cafd1d093f1416be825
Parents: 07fc6e6
Author: Billy Liu 
Authored: Mon Feb 6 23:31:17 2017 +0800
Committer: Billy Liu 
Committed: Mon Feb 6 23:31:17 2017 +0800

--
 atopcalcite/pom.xml |  1 +
 jdbc/pom.xml| 14 ++
 pom.xml |  5 +
 3 files changed, 16 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/96a41592/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1b327fe..c65c0d1 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -43,6 +43,7 @@
 
 
 
+
 
 org.apache.calcite.avatica
 avatica

http://git-wip-us.apache.org/repos/asf/kylin/blob/96a41592/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 34bdc8a..02890a6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -43,7 +43,7 @@
 
 
 org.apache.calcite.avatica
-avatica
+avatica-core
 
 
 junit
@@ -69,9 +69,7 @@
 
 
 
-com.google.protobuf:*
-commons-logging:*
-org.slf4j:jul-to-slf4j
+
org.slf4j:jcl-over-slf4j:*
 
 
 
@@ -85,9 +83,17 @@
 
${shadeBase}.com.fasterxml.jackson
 
 
+com.google.protobuf
+
${shadeBase}.com.google.protobuf
+
+
 org.apache.http
 
${shadeBase}.org.apache.http
 
+
+org.apache.commons
+
${shadeBase}.org.apache.commons
+
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/96a41592/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 3d6c98e..b109783 100644
--- a/pom.xml
+++ b/pom.xml
@@ -524,6 +524,11 @@
 
 
 org.apache.calcite.avatica
+avatica-core
+${avatica.version}
+
+
+org.apache.calcite.avatica
 avatica
 ${avatica.version}
 

kylin git commit: KYLIN-2428 add kylin-tools-init-log4j for get-properties usage

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 6d5ed7ec1 -> f2f0c7eb0


KYLIN-2428 add kylin-tools-init-log4j for get-properties usage


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f2f0c7eb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f2f0c7eb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f2f0c7eb

Branch: refs/heads/KYLIN-2428
Commit: f2f0c7eb0e2c36198569bc2c0b9b016e2e8a50bb
Parents: 6d5ed7e
Author: Billy Liu 
Authored: Wed Feb 8 14:51:43 2017 +0800
Committer: Billy Liu 
Committed: Wed Feb 8 14:51:43 2017 +0800

--
 build/bin/get-properties.sh  |  2 +-
 build/bin/kylin.sh   |  2 +-
 build/conf/kylin-tools-init-log4j.properties | 32 +++
 3 files changed, 34 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f0c7eb/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 170442d..4be9740 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -27,5 +27,5 @@ fi
 
 job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
 tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
-result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 
2>/dev/null`
+result=`java -cp $job_jar:$tool_jar 
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-init-log4j.properties 
org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
 echo "$result"

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f0c7eb/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index abaca62..d95d4ae 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -107,7 +107,7 @@ then
 # KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
 hbase ${KYLIN_EXTRA_START_OPTS} \
 -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
--Dlog4j.configuration=kylin-server-log4j.properties \
+
-Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-server-log4j.properties \
 -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
 -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
 -Djava.endorsed.dirs=${tomcat_root}/endorsed  \

http://git-wip-us.apache.org/repos/asf/kylin/blob/f2f0c7eb/build/conf/kylin-tools-init-log4j.properties
--
diff --git a/build/conf/kylin-tools-init-log4j.properties 
b/build/conf/kylin-tools-init-log4j.properties
new file mode 100644
index 000..650745d
--- /dev/null
+++ b/build/conf/kylin-tools-init-log4j.properties
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#  
+# http://www.apache.org/licenses/LICENSE-2.0
+#  
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+# the kylin-tools-init-log4j.properties is mainly for configuring log 
properties on kylin tools, including:
+#   1. tools launched by get-properties.sh script
+# 
+# It's called kylin-tools-log4j.properties so that it won't distract users 
from the other more important log4j config file: kylin-server-log4j.properties  
+# enable this by -Dlog4j.configuration=kylin-tools-log4j.properties
+
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: 
%m%n
+
+log4j.logger.org.apache.kylin=ERROR

[4/8] kylin git commit: KYLIN-2434 use kylin.source.hive.database-for-flat-table in spark cubing

[billyliu]

KYLIN-2434 use kylin.source.hive.database-for-flat-table in spark cubing


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f62465cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f62465cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f62465cc

Branch: refs/heads/KYLIN-2428
Commit: f62465cc5a5fe0f90142c0606c0c75e8c721bf0e
Parents: b32cc95
Author: lidongsjtu 
Authored: Tue Feb 7 17:31:07 2017 +0800
Committer: lidongsjtu 
Committed: Tue Feb 7 17:31:14 2017 +0800

--
 .../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f62465cc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 8892a73..071806c 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,11 +150,12 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 setupClasspath(sc, confPath);
 HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
 
+final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
+
 HiveContext sqlContext = new HiveContext(sc.sc());
-final DataFrame intermediateTable = sqlContext.table(hiveTable);
+final DataFrame intermediateTable = 
sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + 
hiveTable);
 
 System.setProperty(KylinConfig.KYLIN_CONF, confPath);
-final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
 final CubeInstance cubeInstance = 
CubeManager.getInstance(envConfig).getCube(cubeName);
 final CubeDesc cubeDesc = cubeInstance.getDescriptor();
 final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);

[8/8] kylin git commit: KYLIN-2428 cleanup duplicated shaded libraries

[billyliu]

KYLIN-2428 cleanup duplicated shaded libraries


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d5ed7ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d5ed7ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d5ed7ec

Branch: refs/heads/KYLIN-2428
Commit: 6d5ed7ec16308ec2a2c910710449d0b79e41da84
Parents: 39afa51
Author: Billy Liu 
Authored: Wed Feb 8 14:03:23 2017 +0800
Committer: Billy Liu 
Committed: Wed Feb 8 14:03:23 2017 +0800

--
 atopcalcite/pom.xml |   1 +
 build/bin/kylin.sh  |   4 +-
 core-common/pom.xml |  22 +-
 core-cube/pom.xml   |  18 +-
 .../apache/kylin/gridtable/GTInvertedIndex.java | 223 ---
 .../gridtable/GTInvertedIndexOfColumn.java  | 133 ---
 .../gridtable/SimpleInvertedIndexTest.java  | 196 
 core-job/pom.xml|   9 +
 core-metadata/pom.xml   |  23 ++
 core-storage/pom.xml|  20 ++
 engine-spark/pom.xml|   1 +
 jdbc/pom.xml|  14 +-
 pom.xml | 214 ++
 query/pom.xml   |   6 +
 server/pom.xml  |   4 +
 source-kafka/pom.xml|  16 +-
 storage-hbase/pom.xml   |   4 -
 tool/pom.xml|  33 +++
 18 files changed, 251 insertions(+), 690 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1b327fe..c65c0d1 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -43,6 +43,7 @@
 
 
 
+
 
 org.apache.calcite.avatica
 avatica

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 7813b79..abaca62 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -38,8 +38,8 @@ function retrieveDependency() {
 source ${dir}/setenv.sh
 fi
 
-export 
HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
-export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
+export 
HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+export 
HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}:${KYLIN_HOME}/tool/*
 if [ -n "$KAFKA_HOME" ]
 then
 source ${dir}/find-kafka-dependency.sh

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 016d470..95d3c29 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -35,26 +35,10 @@
 
 
 
-com.fasterxml.jackson.core
-jackson-databind
-
-
-commons-lang
-commons-lang
-
-
 org.apache.commons
 commons-lang3
 
 
-commons-io
-commons-io
-
-
-commons-cli
-commons-cli
-
-
 org.apache.commons
 commons-email
 
@@ -63,9 +47,11 @@
 httpclient
 
 
-com.google.guava
-guava
+com.fasterxml.jackson.core
+jackson-databind
 
+
+
 
 com.google.code.findbugs
 jsr305

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0a30432..fe80783 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -48,9 +48,23 @@
 commons-collections
 commons-collections
 
+
+
+
+commons-lang
+commons-lang
+
+
+commons-io
+commons-io
+
+
+com.google.guava
+guava
+
 
-com.n3twork.druid
-extendedset
+com.google.code.findbugs
+jsr305
 
 
 com.esotericsoftware

[5/8] kylin git commit: KYLIN-2434 minor fix

[billyliu]

KYLIN-2434 minor fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eee9ecbc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eee9ecbc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eee9ecbc

Branch: refs/heads/KYLIN-2428
Commit: eee9ecbcb031862ed316f630933bb320cf93a9d1
Parents: f62465c
Author: lidongsjtu 
Authored: Tue Feb 7 17:37:21 2017 +0800
Committer: lidongsjtu 
Committed: Tue Feb 7 17:37:21 2017 +0800

--
 .../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eee9ecbc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 071806c..317d2bd 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,12 +150,12 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 setupClasspath(sc, confPath);
 HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
 
+System.setProperty(KylinConfig.KYLIN_CONF, confPath);
 final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
 
 HiveContext sqlContext = new HiveContext(sc.sc());
 final DataFrame intermediateTable = 
sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + 
hiveTable);
 
-System.setProperty(KylinConfig.KYLIN_CONF, confPath);
 final CubeInstance cubeInstance = 
CubeManager.getInstance(envConfig).getCube(cubeName);
 final CubeDesc cubeDesc = cubeInstance.getDescriptor();
 final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);

[2/8] kylin git commit: KYLIN-2432 Couldn't select partition column in some old browser

[billyliu]

KYLIN-2432 Couldn't select partition column in some old browser

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d11dd1d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d11dd1d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d11dd1d

Branch: refs/heads/KYLIN-2428
Commit: 6d11dd1d27a74fe50cd4fa58d2acf26f728f82e7
Parents: 8581f1d
Author: luguosheng <550175...@qq.com>
Authored: Tue Feb 7 16:54:05 2017 +0800
Committer: zhongjian 
Committed: Tue Feb 7 17:08:45 2017 +0800

--
 webapp/app/js/config.js | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6d11dd1d/webapp/app/js/config.js
--
diff --git a/webapp/app/js/config.js b/webapp/app/js/config.js
index 5f72eb2..b9ae6a5 100644
--- a/webapp/app/js/config.js
+++ b/webapp/app/js/config.js
@@ -35,7 +35,17 @@ var Config = {
   },
   contact_mail: ''
 };
-
+//resolve startsWith and endsWidth not work in low version chrome
+if (typeof String.prototype.startsWith != 'function') {
+  String.prototype.startsWith = function (prefix){
+return this.slice(0, prefix.length) === prefix;
+  };
+}
+if (typeof String.prototype.endsWith != 'function') {
+  String.prototype.endsWith = function(suffix) {
+return this.indexOf(suffix, this.length - suffix.length) !== -1;
+  };
+}
 // Angular module to load routes.
 KylinApp.config(function ($routeProvider, $httpProvider, $locationProvider, 
$logProvider) {
 //resolve http always use cache data in IE11,IE10

[6/8] kylin git commit: KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob

[billyliu]

KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b860adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b860adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b860adb

Branch: refs/heads/KYLIN-2428
Commit: 7b860adb0a23b53d6e2e40337f4317a3a2b067ba
Parents: eee9ecb
Author: kangkaisen 
Authored: Tue Feb 7 17:35:48 2017 +0800
Committer: kangkaisen 
Committed: Tue Feb 7 17:40:39 2017 +0800

--
 .../kylin/storage/hbase/steps/BulkLoadJob.java | 17 -
 1 file changed, 12 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b860adb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 1c05767..30616c5 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -55,11 +55,18 @@ public class BulkLoadJob extends AbstractHadoopJob {
 
 Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
 FsShell shell = new FsShell(conf);
-try {
-shell.run(new String[] { "-chmod", "-R", "777", input });
-} catch (Exception e) {
-logger.error("Couldn't change the file permissions ", e);
-throw new IOException(e);
+
+int exitCode = -1;
+int retryCount = 10;
+while (exitCode != 0 && retryCount >= 1) {
+exitCode = shell.run(new String[] { "-chmod", "-R", "777", input 
});
+retryCount--;
+Thread.sleep(5000);
+}
+
+if (exitCode != 0) {
+logger.error("Failed to change the file permissions: " + input);
+throw new IOException("Failed to change the file permissions: " + 
input);
 }
 
 String[] newArgs = new String[2];

kylin git commit: update jackson/guava/findbugs to provided

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/dep-clean [created] 7cac866cc


update jackson/guava/findbugs to provided


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7cac866c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7cac866c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7cac866c

Branch: refs/heads/dep-clean
Commit: 7cac866cc3b3b141170b745f9017290b3d7623f8
Parents: 56c7aa5
Author: Billy Liu 
Authored: Sat Feb 4 18:06:49 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 18:06:49 2017 +0800

--
 core-common/pom.xml |   9 +-
 core-cube/pom.xml   |  18 +-
 .../apache/kylin/gridtable/GTInvertedIndex.java | 223 ---
 .../gridtable/GTInvertedIndexOfColumn.java  | 133 ---
 .../gridtable/SimpleInvertedIndexTest.java  | 196 
 core-dictionary/pom.xml |   6 +
 core-job/pom.xml|   5 +
 core-metadata/pom.xml   |  15 ++
 core-storage/pom.xml|  14 ++
 engine-spark/pom.xml|   1 +
 jdbc/pom.xml|   2 -
 pom.xml |  12 +-
 source-kafka/pom.xml|  16 +-
 storage-hbase/pom.xml   |   4 -
 tool/pom.xml|   6 +
 15 files changed, 81 insertions(+), 579 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7cac866c/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 016d470..331b0fe 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -35,10 +35,6 @@
 
 
 
-com.fasterxml.jackson.core
-jackson-databind
-
-
 commons-lang
 commons-lang
 
@@ -62,9 +58,10 @@
 org.apache.httpcomponents
 httpclient
 
+
 
-com.google.guava
-guava
+com.fasterxml.jackson.core
+jackson-databind
 
 
 com.google.code.findbugs

http://git-wip-us.apache.org/repos/asf/kylin/blob/7cac866c/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0a30432..409d531 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -49,14 +49,24 @@
 commons-collections
 
 
-com.n3twork.druid
-extendedset
-
-
 com.esotericsoftware
 kryo-shaded
 
 
+
+
+com.fasterxml.jackson.core
+jackson-databind
+
+
+com.google.guava
+guava
+
+
+com.google.code.findbugs
+jsr305
+
+
 
 
 org.apache.kylin

http://git-wip-us.apache.org/repos/asf/kylin/blob/7cac866c/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
deleted file mode 100644
index a2c713a..000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import 

[kylin] Git Push Summary

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2361 [deleted] 9a3bd71c8

[1/3] kylin git commit: KYLIN-2361 add Tomcat8 ordered class loader

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 19252848e -> 4047e8dc5


KYLIN-2361 add Tomcat8 ordered class loader


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a3bd71c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a3bd71c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a3bd71c

Branch: refs/heads/master
Commit: 9a3bd71c8e5ce9dc13e38560efc556dc862819a1
Parents: 2b60ac6
Author: Billy Liu 
Authored: Sat Feb 4 11:41:49 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 11:42:02 2017 +0800

--
 .../kylin/ext/CustomizedWebappClassloader.java  |   4 +-
 .../kylin/ext/OrderedWebResourceRoot.java   | 286 +++
 .../kylin/ext/WebappOrderedClassLoader.java |  66 +
 3 files changed, 353 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
--
diff --git 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
index f241865..bbf4053 100644
--- 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
+++ 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
@@ -18,14 +18,12 @@
 
 package org.apache.kylin.ext;
 
-import org.apache.catalina.loader.ParallelWebappClassLoader;
-
 /**
  * simple extension to standard ParallelWebappClassLoader
  * the only difference is that CustomizedWebappClassloader is able to delegate 
more packages
  * to parent classloaders
  */
-public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
+public class CustomizedWebappClassloader extends WebappOrderedClassLoader {
 /**
  * Set of package names which are not allowed to be loaded from a webapp
  * class loader without delegating first.

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a3bd71c/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
--
diff --git 
a/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java 
b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
new file mode 100644
index 000..9784bd8
--- /dev/null
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.ext;
+
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.catalina.Context;
+import org.apache.catalina.LifecycleException;
+import org.apache.catalina.LifecycleListener;
+import org.apache.catalina.LifecycleState;
+import org.apache.catalina.TrackedWebResource;
+import org.apache.catalina.WebResource;
+import org.apache.catalina.WebResourceRoot;
+import org.apache.catalina.WebResourceSet;
+
+public class OrderedWebResourceRoot implements WebResourceRoot {
+
+private static final String WEB_INF_LIB_PATH = "/WEB-INF/lib";
+
+private static final Comparator WEB_RESOURCE_COMPARATOR = new 
Comparator() {
+@Override
+public int compare(WebResource o1, WebResource o2) {
+return o1.getName().compareTo(o2.getName());
+}
+};
+
+private WebResourceRoot delegate;
+
+public OrderedWebResourceRoot(WebResourceRoot delegate) {
+this.delegate = delegate;
+}
+
+@Override
+public WebResource[] listResources(String path) {
+WebResource[] webResources = delegate.listResources(path);
+
+if (WEB_INF_LIB_PATH.equals(path)) {
+Arrays.sort(webResources, WEB_RESOURCE_COMPARATOR);
+}
+
+return webResources;
+}
+
+@Override
+public void 

[3/3] kylin git commit: Merge branch 'KYLIN-2361'

[billyliu]

Merge branch 'KYLIN-2361'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4047e8dc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4047e8dc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4047e8dc

Branch: refs/heads/master
Commit: 4047e8dc5bf8aad7c8db79abb5ef2c3be15cd622
Parents: 1925284 9a3bd71
Author: Billy Liu 
Authored: Mon Feb 6 14:37:46 2017 +0800
Committer: Billy Liu 
Committed: Mon Feb 6 14:37:46 2017 +0800

--
 build/script/download-tomcat.sh |   8 +-
 pom.xml |   2 +-
 .../java/org/apache/kylin/rest/DebugTomcat.java |  16 +-
 .../kylin/ext/CustomizedWebappClassloader.java  |   9 +-
 .../kylin/ext/OrderedWebResourceRoot.java   | 286 +++
 .../kylin/ext/WebappOrderedClassLoader.java |  66 +
 6 files changed, 369 insertions(+), 18 deletions(-)
--

[2/3] kylin git commit: KYLIN-2361 Upgrade Tomcat 8.5.9

[billyliu]

KYLIN-2361 Upgrade Tomcat 8.5.9


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b60ac6a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b60ac6a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b60ac6a

Branch: refs/heads/master
Commit: 2b60ac6a42741ca70d63e6680a0fbe9aeed7d46e
Parents: a058bfb
Author: Billy Liu 
Authored: Wed Jan 18 17:19:04 2017 +0800
Committer: Billy Liu 
Committed: Sat Feb 4 11:42:02 2017 +0800

--
 build/script/download-tomcat.sh |  8 
 pom.xml |  2 +-
 .../java/org/apache/kylin/rest/DebugTomcat.java | 16 +---
 .../kylin/ext/CustomizedWebappClassloader.java  |  5 ++---
 4 files changed, 16 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/build/script/download-tomcat.sh
--
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index b3aa509..bdfe351 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -27,19 +27,19 @@ if [[ `uname -a` =~ "Darwin" ]]; then
 alias md5cmd="md5 -q"
 fi
 
-tomcat_pkg_version="7.0.69"
-tomcat_pkg_md5="10a071e5169a1a8b14ff35a0ad181052"
+tomcat_pkg_version="8.5.9"
+tomcat_pkg_md5="b41270a64b7774c964e4bec813eea2ed"
 
 if [ ! -f "build/apache-tomcat-${tomcat_pkg_version}.tar.gz" ]
 then
 echo "no binary file found"
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "Download tomcat failed"
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "Download tomcat failed"
 else
 if [ `md5cmd build/apache-tomcat-${tomcat_pkg_version}.tar.gz | awk 
'{print $1}'` != "${tomcat_pkg_md5}" ]
 then
 echo "md5 check failed"
 rm build/apache-tomcat-${tomcat_pkg_version}.tar.gz
-wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "download tomcat failed"
+wget --directory-prefix=build/ 
http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz
 || echo "download tomcat failed"
 fi
 fi
 unalias md5cmd

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index bf33e07..b82eee2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@
 3.2.4
 2.4.0
 2.5
-7.0.69
+8.5.9
 3.1
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/2b60ac6a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
--
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java 
b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 3461e1d..1b47f79 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -21,11 +21,13 @@ package org.apache.kylin.rest;
 import org.apache.catalina.Context;
 import org.apache.catalina.core.AprLifecycleListener;
 import org.apache.catalina.core.StandardServer;
-import org.apache.catalina.deploy.ErrorPage;
 import org.apache.catalina.startup.Tomcat;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.tomcat.JarScanFilter;
+import org.apache.tomcat.JarScanType;
+import org.apache.tomcat.util.descriptor.web.ErrorPage;
 
 import java.io.File;
 import java.lang.reflect.Field;
@@ -127,12 +129,12 @@ public class DebugTomcat {
 notFound.setLocation("/index.html");
 webContext.addErrorPage(notFound);
 webContext.addWelcomeFile("index.html");
-//webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
-//@Override
-//public boolean check(JarScanType arg0, String arg1) {
-//return false;
-//}
-//});
+webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
+@Override
+public boolean check(JarScanType arg0, String arg1) {
+return false;
+}
+});
 
 // tomcat start
 tomcat.start();

[2/5] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index e0782ce..171e299 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -16,24 +16,6 @@
  */
 package org.apache.calcite.sql2rel;
 
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
 import org.apache.calcite.avatica.util.Spaces;
 import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.plan.Convention;
@@ -85,7 +67,6 @@ import org.apache.calcite.rel.stream.Delta;
 import org.apache.calcite.rel.stream.LogicalDelta;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -116,6 +97,7 @@ import org.apache.calcite.sql.SqlCallBinding;
 import org.apache.calcite.sql.SqlDataTypeSpec;
 import org.apache.calcite.sql.SqlDelete;
 import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlExplainFormat;
 import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.calcite.sql.SqlFunction;
 import org.apache.calcite.sql.SqlIdentifier;
@@ -177,17 +159,37 @@ import org.apache.calcite.util.NumberUtil;
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Util;
 import org.apache.calcite.util.trace.CalciteTrace;
-import org.slf4j.Logger;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+
+import org.slf4j.Logger;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
 
 /*
  * The code has synced with calcite. Hope one day, we could remove the 
hardcode override point.
@@ -209,42 +211,56 @@ import com.google.common.collect.Sets;
 public class SqlToRelConverter {
 //~ Static fields/initializers 
-
 
+protected static final Logger SQL2REL_LOGGER =
+CalciteTrace.getSqlToRelTracer();
+
+private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+
 /** Size of the smallest IN list that will be converted to a semijoin to a
  * static table. */
-public static final int DEFAULT_IN_SUBQUERY_THRESHOLD = 20;
-protected static final Logger SQL2REL_LOGGER = 
CalciteTrace.getSqlToRelTracer();
-private static final BigDecimal TWO = BigDecimal.valueOf(2L);
+public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+
+@Deprecated // to be removed before 2.0
+public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
+DEFAULT_IN_SUB_QUERY_THRESHOLD;
 
 //~ Instance fields 

-public final SqlToRelConverter.Config config;
-public final RelOptTable.ViewExpander viewExpander;
+
 protected final SqlValidator validator;
 protected final RexBuilder rexBuilder;
 protected final Prepare.CatalogReader catalogReader;
 protected final RelOptCluster cluster;
+private DefaultValueFactory defaultValueFactory;
+private SubQueryConverter subQueryConverter;
 protected final List leaves = new ArrayList<>();
-protected final RelDataTypeFactory typeFactory;
 private final List dynamicParamSqlNodes = new 
ArrayList<>();
 

[1/5] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master d264339b1 -> 2c4591a60


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/pom.xml
--
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index cd784f4..874ead6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -66,13 +66,15 @@
 
 
 
false
+
 
-
 
-
com.fasterxml.jackson.core:*
+com.google.protobuf:*
+commons-logging:*
+commons-codec:*
 
 
+
 
 
 org.apache.calcite
@@ -86,10 +88,6 @@
 org.apache.http
 
${shadeBase}.org.apache.http
 
-
-org.apache.commons
-
${shadeBase}.org.apache.commons
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
--
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
index a1b9aef..8e69e68 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java
@@ -209,7 +209,7 @@ public class KylinMeta extends MetaImpl {
 } catch (NoSuchFieldException e) {
 throw new RuntimeException(e);
 }
-columns.add(columnMetaData(name, index, field.getType()));
+columns.add(columnMetaData(name, index, field.getType(), true));
 fields.add(field);
 fieldNames.add(fieldName);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/pom.xml
--
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 080558b..80af108 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -43,6 +43,12 @@
 
 org.apache.kylin
 atopcalcite
+
+
+avatica-core
+org.apache.calcite.avatica
+
+
 
 
 org.apache.kylin
@@ -74,14 +80,6 @@
 org.apache.kylin
 kylin-query
 
-
-org.apache.calcite
-calcite-linq4j
-
-
-commons-cli
-commons-cli
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java 
b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index df6eb2f..05f615f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
 import java.io.File;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
+import java.sql.Driver;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.Statement;

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/pom.xml
--
diff --git a/pom.xml b/pom.xml
index ff4c4e8..57b7752 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,8 +112,8 @@
 1.8.9
 
 
-1.10.0
-1.8.0
+1.11.0
+1.9.0
 
 
 jacoco
@@ -450,15 +450,9 @@
 
 
 
-org.apache.calcite
-calcite-linq4j
-${calcite.version}
-
-
 org.apache.calcite.avatica
 avatica
 ${avatica.version}
-
 
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/query/pom.xml
--
diff --git a/query/pom.xml b/query/pom.xml
index 1dc05d1..6ab74a7 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -36,16 +36,18 @@
 
 org.apache.kylin
 atopcalcite
+
+
+

[5/5] kylin git commit: Merge branch 'KYLIN-2394'

[billyliu]

Merge branch 'KYLIN-2394'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2c4591a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2c4591a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2c4591a6

Branch: refs/heads/master
Commit: 2c4591a609c8d8141c880722a3b8b282b785133e
Parents: d264339 d34f2d3
Author: Billy Liu 
Authored: Sun Jan 22 17:43:16 2017 +0800
Committer: Billy Liu 
Committed: Sun Jan 22 17:43:16 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3782 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2348 insertions(+), 1494 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2c4591a6/kylin-it/pom.xml
--

[4/5] kylin git commit: KYLIN-2394 merge CALCITE-1592

[billyliu]

KYLIN-2394 merge CALCITE-1592


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d34f2d3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d34f2d3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d34f2d3e

Branch: refs/heads/master
Commit: d34f2d3e2bf691c55077a9a76a89217ae915bbdd
Parents: dfb5fac
Author: Billy Liu 
Authored: Fri Jan 20 15:42:37 2017 +0800
Committer: Billy Liu 
Committed: Fri Jan 20 15:42:37 2017 +0800

--
 .../calcite/sql2rel/SqlToRelConverter.java  | 51 +---
 1 file changed, 23 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d34f2d3e/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 171e299..cc9c9a1 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -935,19 +935,21 @@ public class SqlToRelConverter {
 /**
  * Push down all the NOT logical operators into any IN/NOT IN operators.
  *
+ * @param scope Scope where {@code sqlNode} occurs
  * @param sqlNode the root node from which to look for NOT operators
  * @return the transformed SqlNode representation with NOT pushed down.
  */
-private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+private static SqlNode pushDownNotForIn(SqlValidatorScope scope,
+SqlNode sqlNode) {
 if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
 SqlCall sqlCall = (SqlCall) sqlNode;
 if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
 || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
 SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
 for (int i = 0; i < sqlOperands.length; i++) {
-sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+sqlOperands[i] = pushDownNotForIn(scope, sqlOperands[i]);
 }
-return sqlNode;
+return reg(scope, sqlNode);
 } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
 SqlNode childNode = sqlCall.operand(0);
 assert childNode instanceof SqlCall;
@@ -956,48 +958,34 @@ public class SqlToRelConverter {
 SqlNode[] andOperands = childSqlCall.getOperands();
 SqlNode[] orOperands = new SqlNode[andOperands.length];
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-andOperands[i]);
+orOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]));
 }
 for (int i = 0; i < orOperands.length; i++) {
-orOperands[i] = pushDownNotForIn(orOperands[i]);
+orOperands[i] = pushDownNotForIn(scope, orOperands[i]);
 }
-return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
-orOperands[0], orOperands[1]);
+return reg(scope, 
SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], 
orOperands[1]));
 } else if (childSqlCall.getOperator() == 
SqlStdOperatorTable.OR) {
 SqlNode[] orOperands = childSqlCall.getOperands();
 SqlNode[] andOperands = new SqlNode[orOperands.length];
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] =
-SqlStdOperatorTable.NOT.createCall(
-SqlParserPos.ZERO,
-orOperands[i]);
+andOperands[i] = reg(scope, 
SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]));
 }
 for (int i = 0; i < andOperands.length; i++) {
-andOperands[i] = pushDownNotForIn(andOperands[i]);
+andOperands[i] = pushDownNotForIn(scope, 
andOperands[i]);
 }
-return 
SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
- 

[3/5] kylin git commit: KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0

[billyliu]

KYLIN-2394 Upgrade Calcite to 1.11 and Avatica to 1.9.0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dfb5fac9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dfb5fac9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dfb5fac9

Branch: refs/heads/master
Commit: dfb5fac9f0d9c13f1128206b73e683b515fe3d72
Parents: 61833d9
Author: Billy Liu 
Authored: Thu Jan 19 14:54:20 2017 +0800
Committer: Billy Liu 
Committed: Thu Jan 19 14:54:20 2017 +0800

--
 atopcalcite/pom.xml |   11 +-
 .../calcite/sql2rel/SqlToRelConverter.java  | 3941 +++---
 jdbc/pom.xml|   12 +-
 .../java/org/apache/kylin/jdbc/KylinMeta.java   |2 +-
 kylin-it/pom.xml|   14 +-
 .../org/apache/kylin/jdbc/ITJDBCDriverTest.java |1 +
 pom.xml |   10 +-
 query/pom.xml   |   10 +-
 8 files changed, 2430 insertions(+), 1571 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb5fac9/atopcalcite/pom.xml
--
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b916df2..1b327fe 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -36,7 +36,16 @@
 
 org.apache.calcite
 calcite-core
+
+
+org.apache.calcite.avatica
+avatica-core
+
+
+
+
+org.apache.calcite.avatica
+avatica
 
 
-
 

[kylin] Git Push Summary

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2394 [deleted] d34f2d3e2

kylin git commit: KYLIN-1875

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 4c512f00f -> 859605fb3


KYLIN-1875

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/859605fb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/859605fb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/859605fb

Branch: refs/heads/master
Commit: 859605fb35dcfb318a562691cb0ba7a0563db0f1
Parents: 4c512f0
Author: luguosheng <550175...@qq.com>
Authored: Wed Feb 22 16:39:06 2017 +0800
Committer: Billy Liu 
Committed: Wed Feb 22 16:53:43 2017 +0800

--
 webapp/app/js/controllers/cubeEdit.js   | 49 
 webapp/app/js/controllers/cubeSchema.js |  6 +--
 webapp/app/js/controllers/modelDataModel.js |  9 +
 webapp/app/js/model/cubeDescModel.js|  4 +-
 4 files changed, 22 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/859605fb/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 92d8305..2b2d97f 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -40,47 +40,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 supportedEncoding:[],
 encodingMaps:{}
   }
-  TableModel.getColumnTypeEncodingMap().then(function(data){
-$scope.store.encodingMaps=data;
-  });
-  CubeService.getValidEncodings({}, function (encodings) {
-if(encodings){
-  for(var i in encodings)
-if(VdmUtil.isNotExtraKey(encodings,i)){
-  var value = i
-  var name = value;
-  var typeVersion=+encodings[i]||1;
-  var suggest=false,selecttips='';
-  if(/\d+/.test(""+typeVersion)&>=1){
-for(var s=1;s<=typeVersion;s++){
-  if(s==typeVersion){
-suggest=true;
-  }
-  if(value=="int"){
-name = "int (deprecated)";
-suggest=false;
-  }
-  if(typeVersion>1){
-selecttips="(v"+s;
-if(s==typeVersion){
-  selecttips=",suggest)"
-}
-selecttips=')';
-  }
-  $scope.store.supportedEncoding.push({
-"name":name+selecttips,
-"value":value+"[v"+s+"]",
-"version":typeVersion,
-"baseValue":value,
-"suggest":suggest
-  });
-}
-  }
-}
-}
-  },function(e){
-$scope.store.supportedEncoding = $scope.cubeConfig.encodings;
-  })
+
 
   $scope.getColumnsByAlias = function (alias) {
 var temp = [];
@@ -742,6 +702,13 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   return;
 }
 $scope.metaModel.model = modelsManager.getModel(newValue);
+if($scope.metaModel.model){
+  $scope.modelsManager.initAliasMapByModelSchema($scope.metaModel);
+  cubesManager.init();
+  $scope.cubeMetaFrame=CubeDescModel.createNew({
+model_name:newValue
+  })
+}
 if(!$scope.metaModel.model){
   return;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/859605fb/webapp/app/js/controllers/cubeSchema.js
--
diff --git a/webapp/app/js/controllers/cubeSchema.js 
b/webapp/app/js/controllers/cubeSchema.js
index 91a1567..a912c72 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -78,11 +78,11 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, 
QueryService, UserServic
 suggest=false;
   }
   if(typeVersion>1){
-selecttips="(v"+s;
+selecttips=" (v"+s;
 if(s==typeVersion){
-  selecttips=",suggest)"
+  selecttips+=",suggest"
 }
-selecttips=')';
+selecttips+=')';
   }
   $scope.store.supportedEncoding.push({
 "name":name+selecttips,

http://git-wip-us.apache.org/repos/asf/kylin/blob/859605fb/webapp/app/js/controllers/modelDataModel.js
--
diff --git a/webapp/app/js/controllers/modelDataModel.js 
b/webapp/app/js/controllers/modelDataModel.js
index 02c44ea..cb3305a 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -94,6 +94,15 @@ KylinApp.controller('ModelDataModelCtrl', function 
($location,$scope, $modal,cub
 };
   

kylin git commit: KYLIN-1875,remove useless js code

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 859605fb3 -> 6f35b62bd


KYLIN-1875,remove useless js code

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f35b62b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f35b62b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f35b62b

Branch: refs/heads/master
Commit: 6f35b62bd81d793340b944804a4f78e5d8ff7dbe
Parents: 859605f
Author: luguosheng <550175...@qq.com>
Authored: Wed Feb 22 17:01:07 2017 +0800
Committer: Billy Liu 
Committed: Wed Feb 22 17:02:55 2017 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6f35b62b/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 2b2d97f..31b84f4 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -704,7 +704,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 $scope.metaModel.model = modelsManager.getModel(newValue);
 if($scope.metaModel.model){
   $scope.modelsManager.initAliasMapByModelSchema($scope.metaModel);
-  cubesManager.init();
   $scope.cubeMetaFrame=CubeDescModel.createNew({
 model_name:newValue
   })

kylin git commit: KYLIN-2428 Revert SparkEntry to commons-core

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master ed413ee34 -> e05d8f3d8


KYLIN-2428 Revert SparkEntry to commons-core


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e05d8f3d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e05d8f3d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e05d8f3d

Branch: refs/heads/master
Commit: e05d8f3d87e2e7d0acf3bf47daa4e4bbd2d8d75f
Parents: ed413ee
Author: Billy Liu 
Authored: Mon Feb 20 22:26:35 2017 +0800
Committer: Billy Liu 
Committed: Mon Feb 20 22:26:35 2017 +0800

--
 .../apache/kylin/common/util/SparkEntry.java| 46 +++
 .../kylin/engine/spark/SparkExecutable.java |  2 +-
 .../kylin/engine/spark/util/SparkEntry.java | 48 
 3 files changed, 47 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e05d8f3d/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java 
b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
new file mode 100644
index 000..ce11b91
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ */
+public final class SparkEntry {
+
+public static void main(String[] args) throws Exception {
+System.out.println("SparkEntry args:" + StringUtils.join(args, " "));
+if (!(args.length >= 2)) {
+throw new IllegalArgumentException(String.valueOf("-className is 
required"));
+}
+if (!(args[0].equals("-className"))) {
+throw new IllegalArgumentException(String.valueOf("-className is 
required"));
+}
+final String className = args[1];
+final Object o = Class. 
forName(className).newInstance();
+if (!(o instanceof AbstractApplication)) {
+throw new IllegalArgumentException(String.valueOf(className + " is 
not a subClass of AbstractApplication"));
+}
+String[] appArgs = new String[args.length - 2];
+for (int i = 2; i < args.length; i++) {
+appArgs[i - 2] = args[i];
+}
+AbstractApplication abstractApplication = (AbstractApplication) o;
+abstractApplication.execute(appArgs);
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/e05d8f3d/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 015b480..c671a91 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -105,7 +105,7 @@ public class SparkExecutable extends AbstractExecutable {
 }
 
 StringBuilder stringBuilder = new StringBuilder();
-stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit 
--class org.apache.kylin.engine.spark.util.SparkEntry ");
+stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit 
--class org.apache.kylin.common.util.SparkEntry ");
 
 Map sparkConfs = config.getSparkConfigOverride();
 for (Map.Entry entry : sparkConfs.entrySet()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/e05d8f3d/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java 

[kylin] Git Push Summary

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 [deleted] 4f17fa802

kylin git commit: KYLIN-2428 fix commons-lang3 shade in tool assembly

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 4f9400108 -> 385d8011a


KYLIN-2428 fix commons-lang3 shade in tool assembly


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/385d8011
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/385d8011
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/385d8011

Branch: refs/heads/master
Commit: 385d8011a97fdb66cf921933881d3f3e690d75f6
Parents: 4f94001
Author: Billy Liu 
Authored: Sun Feb 19 11:41:58 2017 +0800
Committer: Billy Liu 
Committed: Sun Feb 19 11:41:58 2017 +0800

--
 tool-assembly/pom.xml | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/385d8011/tool-assembly/pom.xml
--
diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml
index 67d92d4..126c0bc 100644
--- a/tool-assembly/pom.xml
+++ b/tool-assembly/pom.xml
@@ -59,6 +59,10 @@
 compile
 
 
+org.apache.commons
+commons-lang3
+
+
 org.slf4j
 slf4j-log4j12
 compile
@@ -93,6 +97,7 @@
 log4j:log4j
 commons-io:commons-io
 
commons-lang:commons-lang
+
org.apache.commons:commons-lang3
 com.google.guava:guava
 org.apache.kylin:*
 

[kylin] Git Push Summary

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 [deleted] 303dea95d

kylin git commit: KYLIN-2428 Move SparkEntry to common-core

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 [created] 4f17fa802


KYLIN-2428 Move SparkEntry to common-core


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f17fa80
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f17fa80
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f17fa80

Branch: refs/heads/KYLIN-2428
Commit: 4f17fa8022064532fda9f939439bdaaa74ac37dc
Parents: 385d801
Author: Billy Liu 
Authored: Mon Feb 20 14:32:35 2017 +0800
Committer: Billy Liu 
Committed: Mon Feb 20 14:32:35 2017 +0800

--
 .../apache/kylin/common/util/SparkEntry.java| 46 +++
 .../kylin/engine/spark/SparkExecutable.java |  2 +-
 .../kylin/engine/spark/util/SparkEntry.java | 48 
 3 files changed, 47 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4f17fa80/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java 
b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
new file mode 100644
index 000..ce11b91
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ */
+public final class SparkEntry {
+
+public static void main(String[] args) throws Exception {
+System.out.println("SparkEntry args:" + StringUtils.join(args, " "));
+if (!(args.length >= 2)) {
+throw new IllegalArgumentException(String.valueOf("-className is 
required"));
+}
+if (!(args[0].equals("-className"))) {
+throw new IllegalArgumentException(String.valueOf("-className is 
required"));
+}
+final String className = args[1];
+final Object o = Class. 
forName(className).newInstance();
+if (!(o instanceof AbstractApplication)) {
+throw new IllegalArgumentException(String.valueOf(className + " is 
not a subClass of AbstractApplication"));
+}
+String[] appArgs = new String[args.length - 2];
+for (int i = 2; i < args.length; i++) {
+appArgs[i - 2] = args[i];
+}
+AbstractApplication abstractApplication = (AbstractApplication) o;
+abstractApplication.execute(appArgs);
+}
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/4f17fa80/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 015b480..c671a91 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -105,7 +105,7 @@ public class SparkExecutable extends AbstractExecutable {
 }
 
 StringBuilder stringBuilder = new StringBuilder();
-stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit 
--class org.apache.kylin.engine.spark.util.SparkEntry ");
+stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit 
--class org.apache.kylin.common.util.SparkEntry ");
 
 Map sparkConfs = config.getSparkConfigOverride();
 for (Map.Entry entry : sparkConfs.entrySet()) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4f17fa80/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java 

kylin git commit: KYLIN-2450 fix uppercase issue in sample.sh

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master 4d08be561 -> a7126f00c


KYLIN-2450 fix uppercase issue in sample.sh


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a7126f00
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a7126f00
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a7126f00

Branch: refs/heads/master
Commit: a7126f00cce821710384153df8cca2867d363f15
Parents: 4d08be5
Author: Billy Liu 
Authored: Sun Feb 19 00:49:29 2017 +0800
Committer: Billy Liu 
Committed: Sun Feb 19 00:49:29 2017 +0800

--
 build/bin/sample.sh   | 14 +-
 .../sample_cube/template/project/learn_kylin.json |  4 +++-
 .../template/table/DEFAULT.KYLIN_CAL_DT.json  |  4 ++--
 3 files changed, 14 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a7126f00/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 74991c0..2c87956 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -30,7 +30,9 @@ hadoop fs -put * /tmp/kylin/sample_cube/data/
 
 hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh 
kylin.source.hive.client`
 sample_database=`bash ${KYLIN_HOME}/bin/get-properties.sh 
kylin.source.hive.database-for-flat-table`
+sample_database=${sample_database^^}
 echo "Going to create sample tables in hive to database "$sample_database" by 
"$hive_client_mode
+
 if [ "${hive_client_mode}" == "beeline" ]
 then
 beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh 
kylin.source.hive.beeline-params`
@@ -39,7 +41,7 @@ then
 beeline ${beeline_params}"/"$sample_database -f 
${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }
 else
 hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database
-hive --database ${sample_database} -f 
${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }
+hive --database $sample_database -f 
${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }
 fi
 
 echo "Sample hive tables are created successfully; Going to create sample 
cube..."
@@ -58,12 +60,14 @@ cp -rf ${KYLIN_HOME}/sample_cube/template/* 
${KYLIN_HOME}/sample_cube/metadata
 
 sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
-sed -i "s/DEFAULT./${sample_database}./g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
-sed -i "s/DEFAULT./${sample_database}./g" 
${KYLIN_HOME}/sample_cube/metadata/model_desc/kylin_sales_model.json
-sed -i "s/DEFAULT./${sample_database}./g" 
${KYLIN_HOME}/sample_cube/metadata/project/learn_kylin.json
 
+ Replace the 'DEFAULT' with kylin.source.hive.database-for-flat-table
+sed -i "s/DEFAULT./$sample_database./g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/DEFAULT./$sample_database./g" 
${KYLIN_HOME}/sample_cube/metadata/model_desc/kylin_sales_model.json
+sed -i "s/DEFAULT./$sample_database./g" 
${KYLIN_HOME}/sample_cube/metadata/project/learn_kylin.json
+sed -i "s/DEFAULT/$sample_database/g" 
${KYLIN_HOME}/sample_cube/metadata/table/*.json
 cd ${KYLIN_HOME}/sample_cube/metadata/table
-ls -1 DEFAULT.KYLIN_*.json|sed "s/\(DEFAULT\)\(.*\)\.json/mv & 
${sample_database}\2.json/"|sh -v
+ls -1 DEFAULT.KYLIN_*.json|sed "s/\(DEFAULT\)\(.*\)\.json/mv & 
$sample_database\2.json/"|sh -v
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a7126f00/examples/sample_cube/template/project/learn_kylin.json
--
diff --git a/examples/sample_cube/template/project/learn_kylin.json 
b/examples/sample_cube/template/project/learn_kylin.json
index e468214..9c96434 100644
--- a/examples/sample_cube/template/project/learn_kylin.json
+++ b/examples/sample_cube/template/project/learn_kylin.json
@@ -11,7 +11,9 @@
   "tables": [
 "DEFAULT.KYLIN_SALES",
 "DEFAULT.KYLIN_CAL_DT",
-"DEFAULT.KYLIN_CATEGORY_GROUPINGS"
+"DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+"DEFAULT.KYLIN_ACCOUNT",
+"DEFAULT.KYLIN_COUNTRY"
   ],
   "models": [
 "kylin_sales_model"

http://git-wip-us.apache.org/repos/asf/kylin/blob/a7126f00/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT.json
--
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT.json 

[1/2] kylin git commit: KYLIN-2428 Cleanup unnecessary shaded libraries for job/coprocessor/jdbc/server

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master a7126f00c -> 4f9400108


http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index 3d33aa8..20b39ee 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -19,15 +19,11 @@
 package org.apache.kylin.engine.spark;
 
 import com.esotericsoftware.kryo.Kryo;
-import com.google.common.collect.Sets;
-import com.google.common.hash.Hashing;
-import org.apache.kylin.measure.MeasureIngester;
 import org.apache.spark.serializer.KryoRegistrator;
-import org.reflections.Reflections;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.Serializable;
+import java.util.LinkedHashSet;
 import java.util.Set;
 
 /**
@@ -39,7 +35,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
 @Override
 public void registerClasses(Kryo kryo) {
 
-Set kyroClasses = Sets.newLinkedHashSet();
+Set kyroClasses = new LinkedHashSet();
 kyroClasses.add(byte[].class);
 kyroClasses.add(int[].class);
 kyroClasses.add(byte[][].class);
@@ -55,15 +51,11 @@ public class KylinKryoRegistrator implements 
KryoRegistrator {
 kyroClasses.add(java.util.HashMap.class);
 kyroClasses.add(java.util.TreeMap.class);
 kyroClasses.add(java.util.Properties.class);
-kyroClasses.addAll(new 
Reflections("org.apache.kylin").getSubTypesOf(Serializable.class));
-kyroClasses.addAll(new 
Reflections("org.apache.kylin.dimension").getSubTypesOf(Serializable.class));
-kyroClasses.addAll(new 
Reflections("org.apache.kylin.cube").getSubTypesOf(Serializable.class));
-kyroClasses.addAll(new 
Reflections("org.apache.kylin.cube.model").getSubTypesOf(Object.class));
-kyroClasses.addAll(new 
Reflections("org.apache.kylin.metadata").getSubTypesOf(Object.class));
-kyroClasses.addAll(new 
Reflections("org.apache.kylin.metadata.model").getSubTypesOf(Object.class));
-kyroClasses.addAll(new 
Reflections("org.apache.kylin.metadata.measure").getSubTypesOf(Object.class));
-kyroClasses.addAll(new 
Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(org.apache.kylin.common.util.BytesSerializer.class));
-kyroClasses.addAll(new 
Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class));
+kyroClasses.add(java.math.MathContext.class);
+kyroClasses.add(java.math.RoundingMode.class);
+kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class);
+kyroClasses.add(java.util.Random.class);
+kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class);
 
 kyroClasses.add(org.apache.spark.sql.Row[].class);
 kyroClasses.add(org.apache.spark.sql.Row.class);
@@ -74,65 +66,29 @@ public class KylinKryoRegistrator implements 
KryoRegistrator {
 kyroClasses.add(org.apache.spark.sql.types.DateType$.class);
 kyroClasses.add(org.apache.spark.sql.types.Metadata.class);
 kyroClasses.add(org.apache.spark.sql.types.StringType$.class);
-kyroClasses.add(Hashing.murmur3_128().getClass());
 
kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class);
 kyroClasses.add(org.apache.spark.sql.types.Decimal.class);
 kyroClasses.add(scala.math.BigDecimal.class);
-kyroClasses.add(java.math.MathContext.class);
-kyroClasses.add(java.math.RoundingMode.class);
-kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class);
-kyroClasses.add(java.util.Random.class);
-kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class);
 
+kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class);
 kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class);
 kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc[].class);
 kyroClasses.add(org.apache.kylin.metadata.model.TblColRef[].class);
-
kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class);
-
kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class);
-
kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class);
-
kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class);
-kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class);
-kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class);
+

[2/2] kylin git commit: KYLIN-2428 Cleanup unnecessary shaded libraries for job/coprocessor/jdbc/server

[billyliu]

KYLIN-2428 Cleanup unnecessary shaded libraries for job/coprocessor/jdbc/server


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f940010
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f940010
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f940010

Branch: refs/heads/master
Commit: 4f940010894a3ee7c284ded15e1d7ef6cc16e966
Parents: a7126f0
Author: Billy Liu 
Authored: Sun Feb 19 00:52:30 2017 +0800
Committer: Billy Liu 
Committed: Sun Feb 19 00:52:30 2017 +0800

--
 assembly/pom.xml|  49 +++-
 atopcalcite/pom.xml |   1 +
 build/bin/get-properties.sh |   3 +-
 build/bin/kylin.sh  |   8 +-
 build/conf/kylin-tools-log4j.properties |   9 +-
 build/script/prepare_libs.sh|   2 +-
 core-common/pom.xml |  22 +-
 .../org/apache/kylin/common/util/ClassUtil.java |  50 
 .../apache/kylin/common/util/SparkEntry.java|  42 ---
 .../apache/kylin/common/util/ClassUtilTest.java |  32 +++
 core-cube/pom.xml   |  18 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   2 +-
 .../apache/kylin/gridtable/GTInvertedIndex.java | 223 
 .../gridtable/GTInvertedIndexOfColumn.java  | 133 --
 .../gridtable/SimpleInvertedIndexTest.java  | 196 --
 core-job/pom.xml|   9 +
 core-metadata/pom.xml   |  23 ++
 .../apache/kylin/dimension/BooleanDimEnc.java   |   5 +-
 .../org/apache/kylin/dimension/DateDimEnc.java  |   4 +-
 .../kylin/dimension/DictionaryDimEnc.java   |   3 +-
 .../apache/kylin/dimension/FixedLenDimEnc.java  |   3 +-
 .../kylin/dimension/FixedLenHexDimEnc.java  |   3 +-
 .../org/apache/kylin/dimension/IntDimEnc.java   |   3 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   3 +-
 .../kylin/dimension/OneMoreByteVLongDimEnc.java |   3 +-
 .../org/apache/kylin/dimension/TimeDimEnc.java  |   4 +-
 .../kylin/measure/MeasureTypeFactory.java   |   2 +-
 .../measure/hllc/HLLDistinctCountAggFunc.java   |   2 +-
 .../org/apache/kylin/measure/topn/Counter.java  |   3 +-
 .../kylin/metadata/model/DataModelDesc.java |   5 +-
 .../apache/kylin/metadata/model/JoinsTree.java  |   2 +-
 .../kylin/metadata/model/PartitionDesc.java |   2 +-
 .../kylin/metadata/model/SegmentStatusEnum.java |   4 +-
 .../apache/kylin/metadata/model/Segments.java   |   3 +-
 .../realization/RealizationStatusEnum.java  |   4 +-
 core-storage/pom.xml|  20 ++
 engine-spark/pom.xml|  47 +---
 .../engine/spark/KylinKryoRegistrator.java  | 265 ++-
 .../spark/SparkBatchCubingJobBuilder2.java  |  21 +-
 .../kylin/engine/spark/SparkCubingByLayer.java  |   6 +-
 .../kylin/engine/spark/SparkExecutable.java |  17 +-
 .../spark/cube/ListBackedCuboidWriter.java  |   4 +-
 .../kylin/engine/spark/util/IteratorUtils.java  |   4 +-
 .../kylin/engine/spark/util/SparkEntry.java |  48 
 .../engine/spark/util/KyroMappingGenerator.java |  56 
 jdbc/pom.xml|  14 +-
 pom.xml | 216 ---
 query/pom.xml   |   6 +
 server/pom.xml  |   4 +
 source-kafka/pom.xml|  16 +-
 storage-hbase/pom.xml   |  19 +-
 tool-assembly/pom.xml   | 135 ++
 tool/pom.xml|  39 ---
 53 files changed, 883 insertions(+), 934 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 873abff..ba69583 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -19,16 +19,20 @@
 
 http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+kylin-assembly
+Apache Kylin - Assembly
+
 
 kylin
 org.apache.kylin
 2.0.0-SNAPSHOT
-
 
-4.0.0
 
-kylin-assembly
-Apache Kylin - Assembly
+
+org.apache.kylin.job.shaded
+
 
 
 
@@ -180,8 +184,45 @@
 org.apache.zookeeper:*
 net.sf.ehcache:*
 
org.apache.httpcomponents:*
+
org.apache.commons:commons-email:*
+  

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c57f3e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c57f3e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c57f3e0

Branch: refs/heads/yang22-hbase1.x
Commit: 1c57f3e0dd6b992b975b78e6bd777d639b0c2f54
Parents: cdb33ca
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Thu Feb 23 15:46:17 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c57f3e0/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[1/3] kylin git commit: KYLIN-2456 fix select * from .. join .. [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase1.x 4853dcae3 -> 1c57f3e0d (forced update)


KYLIN-2456 fix select * from .. join ..


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cdb33ca8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cdb33ca8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cdb33ca8

Branch: refs/heads/yang22-hbase1.x
Commit: cdb33ca8ab4852adb664d3b574bb210d18b2f93c
Parents: 7564274
Author: Li Yang 
Authored: Tue Feb 21 16:13:35 2017 +0800
Committer: Li Yang 
Committed: Tue Feb 21 16:13:35 2017 +0800

--
 .../apache/calcite/sql2rel/SqlToRelConverter.java| 15 ---
 .../org/apache/kylin/query/ITKylinQueryTest.java |  1 +
 2 files changed, 13 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cdb33ca8/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index c70cd20..927bc82 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -602,7 +602,7 @@ public class SqlToRelConverter {
  *   LogicalSort (optional)
  *|- LogicalProject
  *|- LogicalFilter (optional)
- *|- OLAPTableScan
+ *|- OLAPTableScan or LogicalJoin
  */
 LogicalProject rootPrj = null;
 LogicalSort rootSort = null;
@@ -617,8 +617,8 @@ public class SqlToRelConverter {
 
 RelNode input = rootPrj.getInput();
 if (!(//
-input.getClass().getSimpleName().equals("OLAPTableScan")//
-|| 
(input.getClass().getSimpleName().equals("LogicalFilter") && 
input.getInput(0).getClass().getSimpleName().equals("OLAPTableScan"))//
+isAmong(input, "OLAPTableScan", "LogicalJoin")//
+|| (isAmong(input, "LogicalFilter") && 
isAmong(input.getInput(0), "OLAPTableScan", "LogicalJoin"))//
 ))
 return root;
 
@@ -654,6 +654,15 @@ public class SqlToRelConverter {
 return root;
 }
 
+private boolean isAmong(RelNode rel, String... names) {
+String simpleName = rel.getClass().getSimpleName();
+for (String n : names) {
+if (simpleName.equals(n))
+return true;
+}
+return false;
+}
+
 private static boolean isStream(SqlNode query) {
 return query instanceof SqlSelect
 && ((SqlSelect) 
query).isKeywordPresent(SqlSelectKeyword.STREAM);

http://git-wip-us.apache.org/repos/asf/kylin/blob/cdb33ca8/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 98b294d..6f1c629 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -394,5 +394,6 @@ public class ITKylinQueryTest extends KylinTestBase {
 public void testSelectStarColumnCount() throws Exception {
 execAndCompColumnCount("select * from test_kylin_fact limit 10", 9);
 execAndCompColumnCount("select * from test_kylin_fact", 9);
+execAndCompColumnCount("select * from test_kylin_fact left join 
edw.test_cal_dt on test_kylin_fact.cal_dt = edw.test_cal_dt.CAL_DTlimit 
10", 11);
 }
 }

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c57f3e0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c57f3e0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c57f3e0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c57f3e0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[billyliu]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9322c202
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9322c202
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9322c202

Branch: refs/heads/yang22-cdh5.7
Commit: 9322c202f0375c70fdcb96a23c858b7e0063b847
Parents: 1c57f3e
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Billy Liu 
Committed: Thu Feb 23 15:48:22 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9322c202/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/9322c202/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9322c202/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[1/4] kylin git commit: KYLIN-2456 fix select * from .. join .. [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-cdh5.7 9cf1b08f2 -> 9322c202f (forced update)


KYLIN-2456 fix select * from .. join ..


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cdb33ca8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cdb33ca8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cdb33ca8

Branch: refs/heads/yang22-cdh5.7
Commit: cdb33ca8ab4852adb664d3b574bb210d18b2f93c
Parents: 7564274
Author: Li Yang 
Authored: Tue Feb 21 16:13:35 2017 +0800
Committer: Li Yang 
Committed: Tue Feb 21 16:13:35 2017 +0800

--
 .../apache/calcite/sql2rel/SqlToRelConverter.java| 15 ---
 .../org/apache/kylin/query/ITKylinQueryTest.java |  1 +
 2 files changed, 13 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cdb33ca8/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index c70cd20..927bc82 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -602,7 +602,7 @@ public class SqlToRelConverter {
  *   LogicalSort (optional)
  *|- LogicalProject
  *|- LogicalFilter (optional)
- *|- OLAPTableScan
+ *|- OLAPTableScan or LogicalJoin
  */
 LogicalProject rootPrj = null;
 LogicalSort rootSort = null;
@@ -617,8 +617,8 @@ public class SqlToRelConverter {
 
 RelNode input = rootPrj.getInput();
 if (!(//
-input.getClass().getSimpleName().equals("OLAPTableScan")//
-|| 
(input.getClass().getSimpleName().equals("LogicalFilter") && 
input.getInput(0).getClass().getSimpleName().equals("OLAPTableScan"))//
+isAmong(input, "OLAPTableScan", "LogicalJoin")//
+|| (isAmong(input, "LogicalFilter") && 
isAmong(input.getInput(0), "OLAPTableScan", "LogicalJoin"))//
 ))
 return root;
 
@@ -654,6 +654,15 @@ public class SqlToRelConverter {
 return root;
 }
 
+private boolean isAmong(RelNode rel, String... names) {
+String simpleName = rel.getClass().getSimpleName();
+for (String n : names) {
+if (simpleName.equals(n))
+return true;
+}
+return false;
+}
+
 private static boolean isStream(SqlNode query) {
 return query instanceof SqlSelect
 && ((SqlSelect) 
query).isKeywordPresent(SqlSelectKeyword.STREAM);

http://git-wip-us.apache.org/repos/asf/kylin/blob/cdb33ca8/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 98b294d..6f1c629 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -394,5 +394,6 @@ public class ITKylinQueryTest extends KylinTestBase {
 public void testSelectStarColumnCount() throws Exception {
 execAndCompColumnCount("select * from test_kylin_fact limit 10", 9);
 execAndCompColumnCount("select * from test_kylin_fact", 9);
+execAndCompColumnCount("select * from test_kylin_fact left join 
edw.test_cal_dt on test_kylin_fact.cal_dt = edw.test_cal_dt.CAL_DTlimit 
10", 11);
 }
 }

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c57f3e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c57f3e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c57f3e0

Branch: refs/heads/yang22-cdh5.7
Commit: 1c57f3e0dd6b992b975b78e6bd777d639b0c2f54
Parents: cdb33ca
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Thu Feb 23 15:46:17 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c57f3e0/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-

[4/4] kylin git commit: KYLIN-2233 Support HBase 1.0.2

[billyliu]

KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b369f92a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b369f92a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b369f92a

Branch: refs/heads/yang22-hbase102
Commit: b369f92a72c4848cd17a6dd685636c2bc9f701cb
Parents: 1c57f3e
Author: Cheng Wang 
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: Billy Liu 
Committed: Thu Feb 23 15:50:13 2017 +0800

--
 pom.xml   |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++--
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 --
 .../observer/ObserverAggregationCache.java| 10 ++
 .../observer/AggregateRegionObserverTest.java | 10 ++
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b369f92a/pom.xml
--
diff --git a/pom.xml b/pom.xml
index dc23a2f..099316a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
 1.2.1
 
 
-1.1.1
+1.0.2
 0.10.0.0
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b369f92a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean next(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean next(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
 }
 
 @Override
-public boolean nextRaw(List result, ScannerContext scannerContext) 
throws IOException {
+public boolean nextRaw(List result, int limit) throws IOException {
 return next(result);
 }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
 public long getMvccReadPoint() {
 return Long.MAX_VALUE;
 }
-
-@Override
-public int getBatch() {
-return -1;
-}
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/b369f92a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends 
BaseRegionObserver {
 // start/end region operation & sync on scanner is suggested by the
 // javadoc of 

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c57f3e0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c57f3e0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[1/4] kylin git commit: KYLIN-2456 fix select * from .. join .. [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/yang22-hbase102 e1eef9d8a -> b369f92a7 (forced update)


KYLIN-2456 fix select * from .. join ..


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cdb33ca8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cdb33ca8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cdb33ca8

Branch: refs/heads/yang22-hbase102
Commit: cdb33ca8ab4852adb664d3b574bb210d18b2f93c
Parents: 7564274
Author: Li Yang 
Authored: Tue Feb 21 16:13:35 2017 +0800
Committer: Li Yang 
Committed: Tue Feb 21 16:13:35 2017 +0800

--
 .../apache/calcite/sql2rel/SqlToRelConverter.java| 15 ---
 .../org/apache/kylin/query/ITKylinQueryTest.java |  1 +
 2 files changed, 13 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cdb33ca8/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index c70cd20..927bc82 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -602,7 +602,7 @@ public class SqlToRelConverter {
  *   LogicalSort (optional)
  *|- LogicalProject
  *|- LogicalFilter (optional)
- *|- OLAPTableScan
+ *|- OLAPTableScan or LogicalJoin
  */
 LogicalProject rootPrj = null;
 LogicalSort rootSort = null;
@@ -617,8 +617,8 @@ public class SqlToRelConverter {
 
 RelNode input = rootPrj.getInput();
 if (!(//
-input.getClass().getSimpleName().equals("OLAPTableScan")//
-|| 
(input.getClass().getSimpleName().equals("LogicalFilter") && 
input.getInput(0).getClass().getSimpleName().equals("OLAPTableScan"))//
+isAmong(input, "OLAPTableScan", "LogicalJoin")//
+|| (isAmong(input, "LogicalFilter") && 
isAmong(input.getInput(0), "OLAPTableScan", "LogicalJoin"))//
 ))
 return root;
 
@@ -654,6 +654,15 @@ public class SqlToRelConverter {
 return root;
 }
 
+private boolean isAmong(RelNode rel, String... names) {
+String simpleName = rel.getClass().getSimpleName();
+for (String n : names) {
+if (simpleName.equals(n))
+return true;
+}
+return false;
+}
+
 private static boolean isStream(SqlNode query) {
 return query instanceof SqlSelect
 && ((SqlSelect) 
query).isKeywordPresent(SqlSelectKeyword.STREAM);

http://git-wip-us.apache.org/repos/asf/kylin/blob/cdb33ca8/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 98b294d..6f1c629 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -394,5 +394,6 @@ public class ITKylinQueryTest extends KylinTestBase {
 public void testSelectStarColumnCount() throws Exception {
 execAndCompColumnCount("select * from test_kylin_fact limit 10", 9);
 execAndCompColumnCount("select * from test_kylin_fact", 9);
+execAndCompColumnCount("select * from test_kylin_fact left join 
edw.test_cal_dt on test_kylin_fact.cal_dt = edw.test_cal_dt.CAL_DTlimit 
10", 11);
 }
 }

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[billyliu]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c57f3e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c57f3e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c57f3e0

Branch: refs/heads/yang22-hbase102
Commit: 1c57f3e0dd6b992b975b78e6bd777d639b0c2f54
Parents: cdb33ca
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Billy Liu 
Committed: Thu Feb 23 15:46:17 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 12 +--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  4 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 26 +++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 50 files changed, 418 insertions(+), 475 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c57f3e0/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-  

[03/11] kylin git commit: KYLIN-1875,remove useless js code

[billyliu]

KYLIN-1875,remove useless js code

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f35b62b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f35b62b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f35b62b

Branch: refs/heads/master-hbase0.98
Commit: 6f35b62bd81d793340b944804a4f78e5d8ff7dbe
Parents: 859605f
Author: luguosheng <550175...@qq.com>
Authored: Wed Feb 22 17:01:07 2017 +0800
Committer: Billy Liu 
Committed: Wed Feb 22 17:02:55 2017 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6f35b62b/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 2b2d97f..31b84f4 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -704,7 +704,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 $scope.metaModel.model = modelsManager.getModel(newValue);
 if($scope.metaModel.model){
   $scope.modelsManager.initAliasMapByModelSchema($scope.metaModel);
-  cubesManager.init();
   $scope.cubeMetaFrame=CubeDescModel.createNew({
 model_name:newValue
   })

[01/11] kylin git commit: minor improvement on ITFailfastQueryTest [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/master-hbase0.98 da9b080f9 -> f4127f474 (forced update)


minor improvement on ITFailfastQueryTest


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c512f00
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c512f00
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c512f00

Branch: refs/heads/master-hbase0.98
Commit: 4c512f00fc38efa6a211327c3cb32dee4b788007
Parents: 4bdb62c
Author: Hongbin Ma 
Authored: Wed Feb 22 11:44:50 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Feb 22 11:44:50 2017 +0800

--
 .../java/org/apache/kylin/query/ITFailfastQueryTest.java | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4c512f00/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java
index a3720c8..73a597a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java
@@ -67,7 +67,7 @@ public class ITFailfastQueryTest extends KylinTestBase {
 public void testPartitionExceedMaxScanBytes() throws Exception {
 String key = "kylin.storage.partition.max-scan-bytes";
 long saved = 
KylinConfig.getInstanceFromEnv().getPartitionMaxScanBytes();
-KylinConfig.getInstanceFromEnv().setProperty(key, "18000");//very low 
threshold 
+KylinConfig.getInstanceFromEnv().setProperty(key, "1000");//very low 
threshold 
 
 boolean meetExpectedException = false;
 try {
@@ -96,7 +96,7 @@ public class ITFailfastQueryTest extends KylinTestBase {
 public void testPartitionNotExceedMaxScanBytes() throws Exception {
 String key = "kylin.storage.partition.max-scan-bytes";
 long saved = 
KylinConfig.getInstanceFromEnv().getPartitionMaxScanBytes();
-KylinConfig.getInstanceFromEnv().setProperty(key, "2");//enough 
threshold 
+KylinConfig.getInstanceFromEnv().setProperty(key, "10");//enough 
threshold 
 
 try {
 String queryFileName = getQueryFolderPrefix() + 
"src/test/resources/query/sql/query01.sql";
@@ -111,7 +111,7 @@ public class ITFailfastQueryTest extends KylinTestBase {
 public void testQueryExceedMaxScanBytes() throws Exception {
 String key = "kylin.query.max-scan-bytes";
 long saved = KylinConfig.getInstanceFromEnv().getQueryMaxScanBytes();
-KylinConfig.getInstanceFromEnv().setProperty(key, "3");//very low 
threshold 
+KylinConfig.getInstanceFromEnv().setProperty(key, "1000");//very low 
threshold 
 
 boolean meetExpectedException = false;
 try {
@@ -140,7 +140,7 @@ public class ITFailfastQueryTest extends KylinTestBase {
 public void testQueryNotExceedMaxScanBytes() throws Exception {
 String key = "kylin.query.max-scan-bytes";
 long saved = KylinConfig.getInstanceFromEnv().getQueryMaxScanBytes();
-KylinConfig.getInstanceFromEnv().setProperty(key, "4");//enough 
threshold 
+KylinConfig.getInstanceFromEnv().setProperty(key, "10");//enough 
threshold 
 
 try {
 String queryFileName = getQueryFolderPrefix() + 
"src/test/resources/query/sql/query01.sql";

[11/11] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API

[billyliu]

KYLIN-2307 Create a branch for master with HBase 0.98 API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f4127f47
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f4127f47
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f4127f47

Branch: refs/heads/master-hbase0.98
Commit: f4127f47483396adc2dbe6699448a70054255602
Parents: 6cf9749
Author: lidongsjtu 
Authored: Mon Jan 23 13:17:37 2017 +0800
Committer: Billy Liu 
Committed: Thu Feb 23 15:26:07 2017 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |  25 
 dev-support/test_all_against_hdp_2_4_0_0_169.sh |  25 
 .../sandbox/capacity-scheduler.xml  |  17 ++-
 examples/test_case_data/sandbox/core-site.xml   |  28 +---
 examples/test_case_data/sandbox/hbase-site.xml  | 119 +
 examples/test_case_data/sandbox/hdfs-site.xml   |  84 +---
 examples/test_case_data/sandbox/hive-site.xml   |  89 +
 examples/test_case_data/sandbox/mapred-site.xml |  57 +++--
 examples/test_case_data/sandbox/yarn-site.xml   | 127 +++
 .../kylin/provision/BuildCubeWithEngine.java|  17 +--
 pom.xml | 117 +
 .../kylin/rest/security/AclHBaseStorage.java|   4 +-
 .../rest/security/MockAclHBaseStorage.java  |   8 +-
 .../apache/kylin/rest/security/MockHTable.java  |  95 +++---
 .../rest/security/RealAclHBaseStorage.java  |   9 +-
 .../apache/kylin/rest/service/AclService.java   |  25 ++--
 .../apache/kylin/rest/service/CubeService.java  |  35 +++--
 .../apache/kylin/rest/service/QueryService.java |  24 ++--
 .../apache/kylin/rest/service/UserService.java  |  17 +--
 .../kylin/storage/hbase/HBaseConnection.java|  44 +++
 .../kylin/storage/hbase/HBaseResourceStore.java |  31 +++--
 .../storage/hbase/cube/SimpleHBaseStore.java|  20 +--
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |  13 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |   9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |   4 +-
 .../storage/hbase/steps/CubeHTableUtil.java |  16 +--
 .../storage/hbase/steps/DeprecatedGCStep.java   |  24 ++--
 .../storage/hbase/steps/HBaseCuboidWriter.java  |   7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  |  23 ++--
 .../storage/hbase/util/CleanHtableCLI.java  |  12 +-
 .../storage/hbase/util/CubeMigrationCLI.java|  37 +++---
 .../hbase/util/CubeMigrationCheckCLI.java   |  17 +--
 .../hbase/util/DeployCoprocessorCLI.java|  27 ++--
 .../hbase/util/ExtendCubeToHybridCLI.java   |   8 +-
 .../hbase/util/GridTableHBaseBenchmark.java |  34 ++---
 .../kylin/storage/hbase/util/HBaseClean.java|  18 ++-
 .../hbase/util/HBaseRegionSizeCalculator.java   |  35 +++--
 .../kylin/storage/hbase/util/HBaseUsage.java|   9 +-
 .../storage/hbase/util/HbaseStreamingInput.java |  30 ++---
 .../hbase/util/HtableAlterMetadataCLI.java  |   9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java |  19 +--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  |  15 +--
 .../kylin/storage/hbase/util/RowCounterCLI.java |  11 +-
 .../storage/hbase/util/StorageCleanupJob.java   |  20 ++-
 .../storage/hbase/util/UpdateHTableHostCLI.java |  17 +--
 tool/pom.xml|  10 --
 .../org/apache/kylin/tool/CubeMigrationCLI.java |  19 +--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |   8 +-
 48 files changed, 596 insertions(+), 872 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f4127f47/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100755
index 000..f7780dd
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install 

[08/11] kylin git commit: KYLIN-2331 remove unnecessary kylin-spark-conf.propertiesC

[billyliu]

KYLIN-2331 remove unnecessary kylin-spark-conf.propertiesC


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4332b3d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4332b3d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4332b3d4

Branch: refs/heads/master-hbase0.98
Commit: 4332b3d42b631747707231ca47079c230bc628dc
Parents: 37aab3c
Author: Hongbin Ma 
Authored: Thu Feb 23 10:35:06 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Feb 23 14:04:12 2017 +0800

--
 .../sandbox/kylin-spark-conf.properties | 28 
 1 file changed, 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/4332b3d4/examples/test_case_data/sandbox/kylin-spark-conf.properties
--
diff --git a/examples/test_case_data/sandbox/kylin-spark-conf.properties 
b/examples/test_case_data/sandbox/kylin-spark-conf.properties
deleted file mode 100644
index b4a0c97..000
--- a/examples/test_case_data/sandbox/kylin-spark-conf.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-spark.yarn.submit.file.replication=1
-spark.yarn.executor.memoryOverhead=200
-spark.yarn.driver.memoryOverhead=384
-#spark.master=local[4]
-#spark.submit.deployMode=client
-spark.master=yarn
-spark.submit.deployMode=cluster
-spark.eventLog.enabled=true
-spark.yarn.scheduler.heartbeat.interval-ms=5000
-spark.yarn.preserve.staging.files=true
-spark.yarn.queue=default
-spark.yarn.containerLauncherMaxThreads=25
-spark.yarn.max.executor.failures=3
-spark.eventLog.dir=hdfs\:///spark-history
-spark.history.kerberos.enabled=true
-spark.history.provider=org.apache.spark.deploy.history.FsHistoryProvider
-spark.history.ui.port=18080
-spark.history.fs.logDirectory=hdfs\:///spark-history
-spark.executor.memory=1G
-spark.storage.memoryFraction=0.3
-spark.executor.cores=1
-spark.executor.instances=1
-spark.history.kerberos.keytab=none
-spark.history.kerberos.principal=none
-spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
-spark.driver.extraJavaOptions=-Dhdp.version=current
-spark.yarn.am.extraJavaOptions=-Dhdp.version=current
-spark.executor.extraJavaOptions=-Dhdp.version=current

[06/11] kylin git commit: KYLIN 1875 table alias modify

[billyliu]

KYLIN 1875 table alias modify

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/37aab3c6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/37aab3c6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/37aab3c6

Branch: refs/heads/master-hbase0.98
Commit: 37aab3c69ef78633367718a5fd85caa398820f2f
Parents: 77df9dc
Author: chenzhx <346839...@qq.com>
Authored: Wed Feb 22 17:32:14 2017 +0800
Committer: Li Yang 
Committed: Thu Feb 23 11:32:29 2017 +0800

--
 webapp/app/js/controllers/modelDataModel.js | 8 ++--
 1 file changed, 2 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/37aab3c6/webapp/app/js/controllers/modelDataModel.js
--
diff --git a/webapp/app/js/controllers/modelDataModel.js 
b/webapp/app/js/controllers/modelDataModel.js
index cb3305a..15fb20a 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -95,13 +95,9 @@ KylinApp.controller('ModelDataModelCtrl', function 
($location,$scope, $modal,cub
 };
 
 $scope.$watch('newLookup.alias',function(newValue,oldValue){
-  if(!newValue){
-return;
-  }else{
-for(var i=0;i<$scope.newLookup.join.primary_key.length;i++){
-  $scope.newLookup.join.primary_key[i] = 
$scope.newLookup.join.primary_key[i].replace(oldValue+'.',newValue+'.');
+  for(var i=0;i<$scope.newLookup.join.primary_key.length;i++){
+  $scope.newLookup.join.primary_key[i] = 
$scope.newLookup.join.primary_key[i].replace(/^.*?\./,newValue+'.');
 }
-  }
 });
 $scope.editLookup = function (lookup) {
 $scope.lookupState.editingIndex = lookupList.indexOf(lookup);

[05/11] kylin git commit: KYLIN-1875,change model limit remove

[billyliu]

KYLIN-1875,change model limit remove

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/77df9dc1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/77df9dc1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/77df9dc1

Branch: refs/heads/master-hbase0.98
Commit: 77df9dc190c16a3b00c325b245c346722e65eaeb
Parents: 82f6803
Author: luguosheng <550175...@qq.com>
Authored: Wed Feb 22 18:24:14 2017 +0800
Committer: Li Yang 
Committed: Thu Feb 23 11:32:13 2017 +0800

--
 webapp/app/js/controllers/cubeEdit.js | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/77df9dc1/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 31b84f4..0e5038e 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -704,9 +704,11 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 $scope.metaModel.model = modelsManager.getModel(newValue);
 if($scope.metaModel.model){
   $scope.modelsManager.initAliasMapByModelSchema($scope.metaModel);
-  $scope.cubeMetaFrame=CubeDescModel.createNew({
-model_name:newValue
-  })
+  //if(oldValue){
+  //  $scope.cubeMetaFrame=CubeDescModel.createNew({
+  //model_name:newValue
+  //  })
+  //}
 }
 if(!$scope.metaModel.model){
   return;

[11/11] kylin git commit: KYLIN-2428 cleanup unnecessary shade dependency in job/coprocessor/spark/jdbc/tool

[billyliu]

KYLIN-2428 cleanup unnecessary shade dependency in 
job/coprocessor/spark/jdbc/tool


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a5a0904a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a5a0904a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a5a0904a

Branch: refs/heads/KYLIN-2428
Commit: a5a0904a7bba693b5d7d1b6f191c43dbb1cc7270
Parents: c93ea9c
Author: Billy Liu 
Authored: Tue Feb 14 11:18:40 2017 +0800
Committer: Billy Liu 
Committed: Tue Feb 14 11:18:40 2017 +0800

--
 assembly/pom.xml|  49 +++-
 atopcalcite/pom.xml |   1 +
 build/bin/get-properties.sh |   3 +-
 build/bin/kylin.sh  |   8 +-
 build/conf/kylin-tools-log4j.properties |   9 +-
 build/script/prepare_libs.sh|   2 +-
 core-common/pom.xml |  22 +-
 .../apache/kylin/common/util/SparkEntry.java|  42 
 core-cube/pom.xml   |  18 +-
 .../apache/kylin/gridtable/GTInvertedIndex.java | 223 -
 .../gridtable/GTInvertedIndexOfColumn.java  | 133 ---
 .../gridtable/SimpleInvertedIndexTest.java  | 196 ---
 core-job/pom.xml|   9 +
 core-metadata/pom.xml   |  23 ++
 .../apache/kylin/dimension/BooleanDimEnc.java   |   2 +-
 .../kylin/measure/MeasureTypeFactory.java   |   2 +-
 .../measure/hllc/HLLDistinctCountAggFunc.java   |   2 +-
 .../apache/kylin/metadata/model/JoinsTree.java  |   2 +-
 core-storage/pom.xml|  20 ++
 engine-spark/pom.xml|  47 +---
 .../engine/spark/KylinKryoRegistrator.java  | 238 ++-
 .../spark/SparkBatchCubingJobBuilder2.java  |   1 +
 .../kylin/engine/spark/SparkCubingByLayer.java  |   6 +-
 .../kylin/engine/spark/SparkExecutable.java |  17 +-
 .../spark/cube/ListBackedCuboidWriter.java  |   4 +-
 .../kylin/engine/spark/util/IteratorUtils.java  |   4 +-
 .../kylin/engine/spark/util/SparkEntry.java |  48 
 .../engine/spark/util/KyroMappingGenerator.java |  56 +
 jdbc/pom.xml|  14 +-
 pom.xml | 216 +
 query/pom.xml   |   6 +
 server/pom.xml  |   4 +
 source-kafka/pom.xml|  16 +-
 storage-hbase/pom.xml   |  19 +-
 tool-assembly/pom.xml   | 135 +++
 tool/pom.xml|  39 ---
 36 files changed, 734 insertions(+), 902 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a5a0904a/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 873abff..ba69583 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -19,16 +19,20 @@
 
 http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+kylin-assembly
+Apache Kylin - Assembly
+
 
 kylin
 org.apache.kylin
 2.0.0-SNAPSHOT
-
 
-4.0.0
 
-kylin-assembly
-Apache Kylin - Assembly
+
+org.apache.kylin.job.shaded
+
 
 
 
@@ -180,8 +184,45 @@
 org.apache.zookeeper:*
 net.sf.ehcache:*
 
org.apache.httpcomponents:*
+
org.apache.commons:commons-email:*
+
com.sun.mail:javax.mail:*
+
javax.activation:activation:*
 
 
+
+
+com.fasterxml.jackson
+
${shadeBase}.com.fasterxml.jackson
+
+
+com.tdunning
+
${shadeBase}.com.tdunning
+
+
+com.fasterxml.jackson
+
${shadeBase}.com.fasterxml.jackson
+
+
+org.apache.commons.codec
+

[08/11] kylin git commit: KYLIN-2442 calculate raw data size using custom counter

[billyliu]

KYLIN-2442 calculate raw data size using custom counter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/405dee26
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/405dee26
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/405dee26

Branch: refs/heads/KYLIN-2428
Commit: 405dee26d7fe463d15bf5f1d7690359c9e83f678
Parents: 43c0566
Author: Li Yang 
Authored: Fri Feb 10 16:56:21 2017 +0800
Committer: Li Yang 
Committed: Mon Feb 13 10:20:54 2017 +0800

--
 .../kylin/engine/mr/BatchCubingJobBuilder.java  |   1 -
 .../kylin/engine/mr/common/HadoopCmdOutput.java |   9 +-
 .../engine/mr/common/MapReduceExecutable.java   |   8 +-
 .../mr/steps/FactDistinctColumnPartitioner.java |   4 +-
 .../engine/mr/steps/FactDistinctColumnsJob.java |   2 +-
 .../mr/steps/FactDistinctColumnsMapper.java | 262 +++
 .../mr/steps/FactDistinctHiveColumnsMapper.java | 230 
 7 files changed, 272 insertions(+), 244 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/405dee26/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 36c12a1..1ec23b6 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -100,7 +100,6 @@ public class BatchCubingJobBuilder extends 
JobBuilderSupport {
 
 baseCuboidStep.setMapReduceParams(cmd.toString());
 baseCuboidStep.setMapReduceJobClass(BaseCuboidJob.class);
-baseCuboidStep.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + 
CubingJob.SOURCE_SIZE_BYTES);
 return baseCuboidStep;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/405dee26/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index 9d016cc..2a480e6 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -25,6 +25,7 @@ import java.util.Map;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.TaskCounter;
+import 
org.apache.kylin.engine.mr.steps.FactDistinctColumnsMapper.RawDataCounter;
 import org.apache.kylin.job.constant.ExecutableConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -67,7 +68,7 @@ public class HadoopCmdOutput {
 
 private String mapInputRecords;
 private String hdfsBytesWritten;
-private String hdfsBytesRead;
+private String rawInputBytesRead;
 
 public String getMapInputRecords() {
 return mapInputRecords;
@@ -77,8 +78,8 @@ public class HadoopCmdOutput {
 return hdfsBytesWritten;
 }
 
-public String getHdfsBytesRead() {
-return hdfsBytesRead;
+public String getRawInputBytesRead() {
+return rawInputBytesRead;
 }
 
 public void updateJobCounter() {
@@ -95,7 +96,7 @@ public class HadoopCmdOutput {
 
 mapInputRecords = 
String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
 hdfsBytesWritten = 
String.valueOf(counters.findCounter("FileSystemCounters", 
"HDFS_BYTES_WRITTEN").getValue());
-hdfsBytesRead = 
String.valueOf(counters.findCounter("FileSystemCounters", 
"HDFS_BYTES_READ").getValue());
+rawInputBytesRead = 
String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue());
 } catch (Exception e) {
 logger.error(e.getLocalizedMessage(), e);
 output.append(e.getLocalizedMessage());

http://git-wip-us.apache.org/repos/asf/kylin/blob/405dee26/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index 6de07ca..2e7a289 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -202,14 +202,14 @@ public class 

[09/11] kylin git commit: minor, add scan-whole-table flag in TableDesc

[billyliu]

minor, add scan-whole-table flag in TableDesc

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c93ea9c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c93ea9c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c93ea9c9

Branch: refs/heads/KYLIN-2428
Commit: c93ea9c9cb70c285e9b5ce69be3b7b435220ad91
Parents: 405dee2
Author: Cheng Wang 
Authored: Mon Feb 13 16:03:05 2017 +0800
Committer: shaofengshi 
Committed: Mon Feb 13 18:54:10 2017 +0800

--
 .../org/apache/kylin/metadata/model/TableDesc.java  | 16 +---
 .../org/apache/kylin/source/hive/HiveMRInput.java   |  5 +++--
 2 files changed, 16 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c93ea9c9/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index e4fc4c3..be62062 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -26,8 +26,8 @@ import 
org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.StringSplitter;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
@@ -46,9 +46,11 @@ public class TableDesc extends RootPersistentEntity 
implements ISourceAware {
 private ColumnDesc[] columns;
 @JsonProperty("source_type")
 private int sourceType = ISourceAware.ID_HIVE;
+@JsonProperty("whole_scan")
+private boolean wholeScan = true;
 @JsonProperty("table_type")
 private String tableType;
-
+
 @JsonProperty("data_gen")
 @JsonInclude(JsonInclude.Include.NON_NULL)
 private String dataGen;
@@ -101,7 +103,15 @@ public class TableDesc extends RootPersistentEntity 
implements ISourceAware {
 }
 return identity;
 }
-
+
+public void setWholeScan(boolean wholeScan) {
+this.wholeScan = wholeScan;
+}
+
+public boolean getWholeScan() {
+return this.wholeScan;
+}
+
 public boolean isView() {
 return TABLE_TYPE_VIRTUAL_VIEW.equals(tableType);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c93ea9c9/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index d665dc7..f9c7cfc 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.util.Map;
 import java.util.Set;
 
-import com.google.common.collect.Sets;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -57,10 +56,12 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.Sets;
+
 public class HiveMRInput implements IMRInput {
 
 public static String getTableNameForHCat(TableDesc table) {
-String tableName = table.isView() ? table.getMaterializedName() : 
table.getName();
+String tableName = (table.isView() || !table.getWholeScan()) ? 
table.getMaterializedName() : table.getName();
 return String.format("%s.%s", table.getDatabase(), 
tableName).toUpperCase();
 }
 

[06/11] kylin git commit: KYLIN-2443 Report coprocessor error information back to client

[billyliu]

http://git-wip-us.apache.org/repos/asf/kylin/blob/43c05667/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index 4b6fc95..b9f2771 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -121,10 +121,18 @@ public final class CubeVisitProtos {
 // optional int64 maxScanBytes = 8;
 /**
  * optional int64 maxScanBytes = 8;
+ *
+ * 
+ * 0 means no limit
+ * 
  */
 boolean hasMaxScanBytes();
 /**
  * optional int64 maxScanBytes = 8;
+ *
+ * 
+ * 0 means no limit
+ * 
  */
 long getMaxScanBytes();
   }
@@ -972,12 +980,20 @@ public final class CubeVisitProtos {
 private long maxScanBytes_;
 /**
  * optional int64 maxScanBytes = 8;
+ *
+ * 
+ * 0 means no limit
+ * 
  */
 public boolean hasMaxScanBytes() {
   return ((bitField0_ & 0x0040) == 0x0040);
 }
 /**
  * optional int64 maxScanBytes = 8;
+ *
+ * 
+ * 0 means no limit
+ * 
  */
 public long getMaxScanBytes() {
   return maxScanBytes_;
@@ -2050,18 +2066,30 @@ public final class CubeVisitProtos {
   private long maxScanBytes_ ;
   /**
* optional int64 maxScanBytes = 8;
+   *
+   * 
+   * 0 means no limit
+   * 
*/
   public boolean hasMaxScanBytes() {
 return ((bitField0_ & 0x0080) == 0x0080);
   }
   /**
* optional int64 maxScanBytes = 8;
+   *
+   * 
+   * 0 means no limit
+   * 
*/
   public long getMaxScanBytes() {
 return maxScanBytes_;
   }
   /**
* optional int64 maxScanBytes = 8;
+   *
+   * 
+   * 0 means no limit
+   * 
*/
   public Builder setMaxScanBytes(long value) {
 bitField0_ |= 0x0080;
@@ -2071,6 +2099,10 @@ public final class CubeVisitProtos {
   }
   /**
* optional int64 maxScanBytes = 8;
+   *
+   * 
+   * 0 means no limit
+   * 
*/
   public Builder clearMaxScanBytes() {
 bitField0_ = (bitField0_ & ~0x0080);
@@ -2116,6 +2148,32 @@ public final class CubeVisitProtos {
  * required .CubeVisitResponse.Stats stats = 2;
  */
 
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.StatsOrBuilder
 getStatsOrBuilder();
+
+// optional .CubeVisitResponse.ErrorInfo errorInfo = 3;
+/**
+ * optional .CubeVisitResponse.ErrorInfo errorInfo = 3;
+ *
+ * 
+ * should be set when stats.normalComplete == false
+ * 
+ */
+boolean hasErrorInfo();
+/**
+ * optional .CubeVisitResponse.ErrorInfo errorInfo = 3;
+ *
+ * 
+ * should be set when stats.normalComplete == false
+ * 
+ */
+
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.ErrorInfo
 getErrorInfo();
+/**
+ * optional .CubeVisitResponse.ErrorInfo errorInfo = 3;
+ *
+ * 
+ * should be set when stats.normalComplete == false
+ * 
+ */
+
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.ErrorInfoOrBuilder
 getErrorInfoOrBuilder();
   }
   /**
* Protobuf type {@code CubeVisitResponse}
@@ -2186,6 +2244,19 @@ public final class CubeVisitProtos {
   bitField0_ |= 0x0002;
   break;
 }
+case 26: {
+  
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.ErrorInfo.Builder
 subBuilder = null;
+  if (((bitField0_ & 0x0004) == 0x0004)) {
+subBuilder = errorInfo_.toBuilder();
+  }
+  errorInfo_ = 
input.readMessage(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.ErrorInfo.PARSER,
 extensionRegistry);
+  if (subBuilder != null) {
+subBuilder.mergeFrom(errorInfo_);
+errorInfo_ = subBuilder.buildPartial();
+  }
+  bitField0_ |= 0x0004;
+  break;
+}
   }
 }
   } catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -2225,6 +2296,97 @@ public final class CubeVisitProtos {
   return PARSER;
 }
 
+/**
+ * Protobuf enum {@code 

[04/11] kylin git commit: minor, make FileResourceStore synchronized

[billyliu]

minor, make FileResourceStore synchronized


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd4e116a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd4e116a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd4e116a

Branch: refs/heads/KYLIN-2428
Commit: cd4e116ac3391d4da975a29c21a0c69f053463a6
Parents: 3c49c9e
Author: Yang Li 
Authored: Fri Feb 10 21:41:50 2017 +0800
Committer: Yang Li 
Committed: Fri Feb 10 21:41:50 2017 +0800

--
 .../common/persistence/FileResourceStore.java | 18 +-
 1 file changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cd4e116a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index 3e012f5..d84e587 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -50,7 +50,7 @@ public class FileResourceStore extends ResourceStore {
 }
 
 @Override
-protected NavigableSet listResourcesImpl(String folderPath) throws 
IOException {
+synchronized protected NavigableSet listResourcesImpl(String 
folderPath) throws IOException {
 String[] names = file(folderPath).list();
 if (names == null) // not a directory
 return null;
@@ -64,13 +64,13 @@ public class FileResourceStore extends ResourceStore {
 }
 
 @Override
-protected boolean existsImpl(String resPath) throws IOException {
+synchronized protected boolean existsImpl(String resPath) throws 
IOException {
 File f = file(resPath);
 return f.exists() && f.isFile(); // directory is not considered a 
resource
 }
 
 @Override
-protected List getAllResourcesImpl(String folderPath, long 
timeStart, long timeEndExclusive) throws IOException {
+synchronized protected List getAllResourcesImpl(String 
folderPath, long timeStart, long timeEndExclusive) throws IOException {
 NavigableSet resources = listResources(folderPath);
 if (resources == null)
 return Collections.emptyList();
@@ -95,7 +95,7 @@ public class FileResourceStore extends ResourceStore {
 }
 
 @Override
-protected RawResource getResourceImpl(String resPath) throws IOException {
+synchronized protected RawResource getResourceImpl(String resPath) throws 
IOException {
 File f = file(resPath);
 if (f.exists() && f.isFile()) {
 if (f.length() == 0) {
@@ -108,7 +108,7 @@ public class FileResourceStore extends ResourceStore {
 }
 
 @Override
-protected long getResourceTimestampImpl(String resPath) throws IOException 
{
+synchronized protected long getResourceTimestampImpl(String resPath) 
throws IOException {
 File f = file(resPath);
 if (f.exists() && f.isFile())
 return f.lastModified();
@@ -117,7 +117,7 @@ public class FileResourceStore extends ResourceStore {
 }
 
 @Override
-protected void putResourceImpl(String resPath, InputStream content, long 
ts) throws IOException {
+synchronized protected void putResourceImpl(String resPath, InputStream 
content, long ts) throws IOException {
 File f = file(resPath);
 f.getParentFile().mkdirs();
 FileOutputStream out = new FileOutputStream(f);
@@ -131,7 +131,7 @@ public class FileResourceStore extends ResourceStore {
 }
 
 @Override
-protected long checkAndPutResourceImpl(String resPath, byte[] content, 
long oldTS, long newTS) throws IOException, IllegalStateException {
+synchronized protected long checkAndPutResourceImpl(String resPath, byte[] 
content, long oldTS, long newTS) throws IOException, IllegalStateException {
 File f = file(resPath);
 if ((f.exists() && f.lastModified() != oldTS) || (f.exists() == false 
&& oldTS != 0))
 throw new IllegalStateException("Overwriting conflict " + resPath 
+ ", expect old TS " + oldTS + ", but found " + f.lastModified());
@@ -143,13 +143,13 @@ public class FileResourceStore extends ResourceStore {
 }
 
 @Override
-protected void deleteResourceImpl(String resPath) throws IOException {
+synchronized protected void deleteResourceImpl(String resPath) throws 
IOException {
 File f = file(resPath);
 f.delete();
 }
 
 @Override
-protected String getReadableResourcePathImpl(String resPath) {
+synchronized protected String