[3/3] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/20df2afb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/20df2afb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/20df2afb

Branch: refs/heads/yang21-cdh5.7
Commit: 20df2afbbcfc205f69ca7f042cfd0a836237f5eb
Parents: c1d707e
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Yang Li 
Committed: Tue Oct 4 12:53:31 2016 +

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/20df2afb/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/20df2afb/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/20df2afb/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1d707e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1d707e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1d707e7

Branch: refs/heads/yang21-cdh5.7
Commit: c1d707e7def6dad3af4d3773b99c5a1ff30b62a8
Parents: 85c2754
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 12:51:33 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c1d707e7/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF 

[1/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 da1449262 -> 20df2afbb (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/c1d707e7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c1d707e7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1d707e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1d707e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1d707e7

Branch: refs/heads/yang21-hbase1.x
Commit: c1d707e7def6dad3af4d3773b99c5a1ff30b62a8
Parents: 85c2754
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 12:51:33 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c1d707e7/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 67158a94b -> c1d707e7d (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/c1d707e7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c1d707e7/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/67158a94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/67158a94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/67158a94

Branch: refs/heads/yang21-hbase1.x
Commit: 67158a94b09fb2da620edaf3dd435f6a22b1427e
Parents: 85c2754
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 12:29:34 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/67158a94/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[1/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x d766919dc -> 67158a94b (forced update)


http://git-wip-us.apache.org/repos/asf/kylin/blob/67158a94/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/67158a94/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 

[2/6] kylin git commit: minor, fix sample cube desc

[liyang]

minor, fix sample cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e27e54e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e27e54e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e27e54e7

Branch: refs/heads/yang21-cdh5.7
Commit: e27e54e78a29bb4694ea2271b2f9c77a7cd61fec
Parents: 7478d1b
Author: Yang Li 
Authored: Fri Sep 30 21:46:31 2016 +0800
Committer: Yang Li 
Committed: Fri Sep 30 21:46:31 2016 +0800

--
 .../org/apache/kylin/engine/mr/HadoopUtil.java  |  3 +-
 .../cube_desc/kylin_sales_cube_desc.json| 46 ++--
 2 files changed, 24 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e27e54e7/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 0ed39d5..88692a0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -34,6 +34,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class HadoopUtil {
+@SuppressWarnings("unused")
 private static final Logger logger = 
LoggerFactory.getLogger(HadoopUtil.class);
 private static final ThreadLocal hadoopConfig = new 
ThreadLocal<>();
 
@@ -45,11 +46,9 @@ public class HadoopUtil {
 if (hadoopConfig.get() == null) {
 Configuration conf = healSickConfig(new Configuration());
 // do not cache this conf, or will affect following mr jobs
-logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 return conf;
 }
 Configuration conf = hadoopConfig.get();
-logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 return conf;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/e27e54e7/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index 70b1294..99f8f46 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -1,7 +1,10 @@
 {
   "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
   "name" : "kylin_sales_cube_desc",
+  "model_name" : "kylin_sales_model",
   "description" : null,
+  "null_string" : null,
   "dimensions" : [ {
 "name" : "CAL_DT",
 "table" : "DEFAULT.KYLIN_CAL_DT",
@@ -106,21 +109,21 @@
 },
 "dependent_measure_ref" : null
   }, {
-"name": "TOP_SELLER",
-"function": {
-  "expression": "TOP_N",
-  "parameter": {
-"type": "column",
-"value": "PRICE",
-"next_parameter": {
-  "type": "column",
-  "value": "SELLER_ID",
-  "next_parameter": null
+"name" : "TOP_SELLER",
+"function" : {
+  "expression" : "TOP_N",
+  "parameter" : {
+"type" : "column",
+"value" : "PRICE",
+"next_parameter" : {
+  "type" : "column",
+  "value" : "SELLER_ID",
+  "next_parameter" : null
 }
   },
-  "returntype": "topn(100)"
+  "returntype" : "topn(100)"
 },
-"dependent_measure_ref": null
+"dependent_measure_ref" : null
   } ],
   "rowkey" : {
 "rowkey_columns" : [ {
@@ -146,21 +149,18 @@
   "encoding" : "dict"
 } ]
   },
-  "last_modified" : 1451468470824,
-  "model_name" : "kylin_sales_model",
-  "null_string" : null,
   "hbase_mapping" : {
 "column_family" : [ {
-  "name" : "f1",
+  "name" : "F1",
   "columns" : [ {
-"qualifier" : "m",
-"measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
+"qualifier" : "M",
+"measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", 
"TOP_SELLER" ]
   } ]
 }, {
-  "name" : "f2",
+  "name" : "F2",
   "columns" : [ {
-"qualifier" : "m",
-"measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT", "TOP_SELLER" 
]
+"qualifier" : "M",
+"measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT" ]
   } ]
 } ]
   },
@@ -174,10 +174,10 @@
   } ],
   "notify_list" : null,
   "status_need_notify" : [ ],
+  "partition_date_start" : 132537600,
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
   "engine_type" : %default_engine_type%,

[5/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d766919d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d766919d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d766919d

Branch: refs/heads/yang21-cdh5.7
Commit: d766919dc2d83ca7207b5bf79b748ede6da3f54c
Parents: 85c2754
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 11:25:44 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d766919d/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF 

[6/6] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da144926
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da144926
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da144926

Branch: refs/heads/yang21-cdh5.7
Commit: da1449262d05fdfc22894f2d3357e01665398cbb
Parents: d766919
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Yang Li 
Committed: Tue Oct 4 12:00:56 2016 +

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/da144926/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/da144926/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/da144926/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[3/6] kylin git commit: minor, clean error handling on Job classes

[liyang]

minor, clean error handling on Job classes


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85c27542
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85c27542
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85c27542

Branch: refs/heads/yang21-cdh5.7
Commit: 85c27542f4d53c05fc3adc834679760252fb40d6
Parents: e27e54e
Author: Yang Li 
Authored: Fri Sep 30 22:05:12 2016 +0800
Committer: Yang Li 
Committed: Fri Sep 30 22:05:12 2016 +0800

--
 .../engine/mr/steps/CreateDictionaryJob.java| 34 -
 .../apache/kylin/engine/mr/steps/CuboidJob.java |  4 --
 .../engine/mr/steps/FactDistinctColumnsJob.java |  4 --
 .../kylin/engine/mr/steps/InMemCuboidJob.java   |  4 --
 .../kylin/engine/mr/steps/MergeCuboidJob.java   |  4 --
 .../engine/mr/steps/MetadataCleanupJob.java | 21 +++---
 .../mr/steps/RowKeyDistributionCheckerJob.java  | 59 +++
 .../cardinality/HiveColumnCardinalityJob.java   | 76 +---
 .../kylin/storage/hbase/steps/BulkLoadJob.java  | 54 +++---
 .../storage/hbase/steps/CreateHTableJob.java| 27 +++
 .../kylin/storage/hbase/steps/CubeHFileJob.java |  4 --
 11 files changed, 118 insertions(+), 173 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/85c27542/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 69c0095..5db7c88 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -40,29 +40,23 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
 @Override
 public int run(String[] args) throws Exception {
 Options options = new Options();
+options.addOption(OPTION_CUBE_NAME);
+options.addOption(OPTION_SEGMENT_ID);
+options.addOption(OPTION_INPUT_PATH);
+parseOptions(options, args);
 
-try {
-options.addOption(OPTION_CUBE_NAME);
-options.addOption(OPTION_SEGMENT_ID);
-options.addOption(OPTION_INPUT_PATH);
-parseOptions(options, args);
+final String cubeName = getOptionValue(OPTION_CUBE_NAME);
+final String segmentID = getOptionValue(OPTION_SEGMENT_ID);
+final String factColumnsInputPath = getOptionValue(OPTION_INPUT_PATH);
 
-final String cubeName = getOptionValue(OPTION_CUBE_NAME);
-final String segmentID = getOptionValue(OPTION_SEGMENT_ID);
-final String factColumnsInputPath = 
getOptionValue(OPTION_INPUT_PATH);
+KylinConfig config = KylinConfig.getInstanceFromEnv();
 
-KylinConfig config = KylinConfig.getInstanceFromEnv();
-
-DictionaryGeneratorCLI.processSegment(config, cubeName, segmentID, 
new DistinctColumnValuesProvider() {
-@Override
-public ReadableTable getDistinctValuesFor(TblColRef col) {
-return new DFSFileTable(factColumnsInputPath + "/" + 
col.getName(), -1);
-}
-});
-} catch (Exception e) {
-printUsage(options);
-throw e;
-}
+DictionaryGeneratorCLI.processSegment(config, cubeName, segmentID, new 
DistinctColumnValuesProvider() {
+@Override
+public ReadableTable getDistinctValuesFor(TblColRef col) {
+return new DFSFileTable(factColumnsInputPath + "/" + 
col.getName(), -1);
+}
+});
 
 return returnCode;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/85c27542/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index 0399300..6b0c86e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -140,10 +140,6 @@ public class CuboidJob extends AbstractHadoopJob {
 this.deletePath(job.getConfiguration(), output);
 
 return waitForCompletion(job);
-} catch (Exception e) {
-logger.error("error in CuboidJob", e);
-printUsage(options);
-throw e;
 } finally {
 if (job != null)
 

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/d766919d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/d766919d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[1/6] kylin git commit: minor, retry when http failed in job diagnosis [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 d5efd44c0 -> da1449262 (forced update)


minor, retry when http failed in job diagnosis


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7478d1b6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7478d1b6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7478d1b6

Branch: refs/heads/yang21-cdh5.7
Commit: 7478d1b6a116d3cb24301b272197b60294f2bb15
Parents: e10f2b9
Author: lidongsjtu 
Authored: Thu Sep 29 09:48:58 2016 +0800
Committer: lidongsjtu 
Committed: Thu Sep 29 17:11:09 2016 +0800

--
 .../java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java | 4 ++--
 .../java/org/apache/kylin/tool/JobTaskCounterExtractor.java | 9 +++--
 2 files changed, 9 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7478d1b6/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java 
b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
index 635a2c3..c8ff1f4 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
@@ -99,8 +99,8 @@ public class JobDiagnosisInfoCLI extends 
AbstractInfoExtractor {
 addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + jobId);
 addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + jobId);
 for (ExecutablePO task : executablePO.getTasks()) {
-addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + 
executablePO.getTasks());
-addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + 
executablePO.getTasks());
+addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + 
task.getUuid());
+addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + 
task.getUuid());
 if (includeYarnLogs) {
 yarnLogsResources.add(task.getUuid());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7478d1b6/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
--
diff --git 
a/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java 
b/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
index 7b9d8f3..6a317e9 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
@@ -47,6 +47,8 @@ public class JobTaskCounterExtractor extends 
AbstractInfoExtractor {
 private String yarnUrl;
 private static final Logger logger = 
LoggerFactory.getLogger(JobTaskCounterExtractor.class);
 
+private final int HTTP_RETRY = 3;
+
 public JobTaskCounterExtractor(String mrJobId) {
 this.mrJobId = mrJobId;
 this.yarnUrl = getRestCheckUrl();
@@ -87,14 +89,17 @@ public class JobTaskCounterExtractor extends 
AbstractInfoExtractor {
 private String getHttpResponse(String url) {
 HttpClient client = new HttpClient();
 String response = null;
-while (response == null) {
+int retry_times = 0;
+while (response == null && retry_times < HTTP_RETRY) {
+retry_times++;
+
 HttpMethod get = new GetMethod(url);
 try {
 get.addRequestHeader("accept", "application/json");
 client.executeMethod(get);
 response = get.getResponseBodyAsString();
 } catch (Exception e) {
-logger.warn("Failed to fetch http response" + e);
+logger.warn("Failed to fetch http response. Retry={}", 
retry_times, e);
 } finally {
 get.releaseConnection();
 }

[2/2] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d766919d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d766919d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d766919d

Branch: refs/heads/yang21-hbase1.x
Commit: d766919dc2d83ca7207b5bf79b748ede6da3f54c
Parents: 85c2754
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 11:25:44 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d766919d/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[kylin] Git Push Summary

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 [created] d5efd44c0

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3d38f042
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3d38f042
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3d38f042

Branch: refs/heads/master-cdh5.7
Commit: 3d38f042eba4f1f1630e70d7724d14f7be5f2ff0
Parents: 792d4ee
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 08:56:35 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 461 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3d38f042/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF 

[4/4] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5efd44c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5efd44c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5efd44c

Branch: refs/heads/master-cdh5.7
Commit: d5efd44c0ae6c70fe15820bd9b244ca156c63459
Parents: 3d38f04
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Yang Li 
Committed: Tue Oct 4 08:57:33 2016 +

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d5efd44c/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5efd44c/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d5efd44c/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[1/4] kylin git commit: minor, enhance dev-support scripts [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master-cdh5.7 59beaa0ed -> d5efd44c0 (forced update)


minor, enhance dev-support scripts


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/792d4ee3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/792d4ee3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/792d4ee3

Branch: refs/heads/master-cdh5.7
Commit: 792d4ee3fd3f12b63a34a47bb90ed7b87449f15a
Parents: 937cc54
Author: Yang Li 
Authored: Tue Oct 4 16:55:46 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 16:55:46 2016 +0800

--
 dev-support/sync_hbase_cdh_branches.cmd | 43 --
 dev-support/sync_hbase_cdh_branches.sh  | 45 
 2 files changed, 45 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/792d4ee3/dev-support/sync_hbase_cdh_branches.cmd
--
diff --git a/dev-support/sync_hbase_cdh_branches.cmd 
b/dev-support/sync_hbase_cdh_branches.cmd
deleted file mode 100644
index ed828ca..000
--- a/dev-support/sync_hbase_cdh_branches.cmd
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/bash
-
-# 
-
-base=master
-
-# 
-
-set -o pipefail  # trace ERR through pipes
-set -o errtrace  # trace ERR through 'time command' and other functions
-function error() {
-   SCRIPT="$0"   # script name
-   LASTLINE="$1" # line of error occurrence
-   LASTERR="$2"  # error code
-   echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code 
${LASTERR}"
-   exit 1
-}
-trap 'error ${LINENO} ${?}' ERR
-
-# 
-
-git fetch apache
-git checkout apache/$base-hbase1.x
-git format-patch -1
-git checkout apache/$base-cdh1.x
-git format-patch -1
-
-git checkout apache/$base
-git checkout -b tmp
-git reset apache/$base --hard
-
-git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-#git push apache tmp:$base-hbase1.x -f
-rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-
-git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
-#git push apache tmp:$base-cdh5.7 -f
-rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
-
-# clean up
-git checkout master
-git reset apache/master --hard
-git checkout -b tmp

http://git-wip-us.apache.org/repos/asf/kylin/blob/792d4ee3/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh 
b/dev-support/sync_hbase_cdh_branches.sh
new file mode 100644
index 000..f52284a
--- /dev/null
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# 
+
+base=master
+
+# 
+
+set -o pipefail  # trace ERR through pipes
+set -o errtrace  # trace ERR through 'time command' and other functions
+function error() {
+   SCRIPT="$0"   # script name
+   LASTLINE="$1" # line of error occurrence
+   LASTERR="$2"  # error code
+   echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code 
${LASTERR}"
+   exit 1
+}
+trap 'error ${LINENO} ${?}' ERR
+
+# 
+
+git fetch apache
+git checkout apache/$base-hbase1.x
+git format-patch -1
+git checkout apache/$base-cdh5.7
+git format-patch -1
+
+git checkout apache/$base
+git checkout -b tmp
+git reset apache/$base --hard
+
+git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
+mvn clean compile -DskipTests
+git push apache tmp:$base-hbase1.x -f
+rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
+
+git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+mvn clean compile -DskipTests
+git push apache tmp:$base-cdh5.7 -f
+rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+
+# clean up
+git checkout master
+git reset apache/master --hard
+git branch -D tmp

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/3d38f042/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3d38f042/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3d38f042
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3d38f042
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3d38f042

Branch: refs/heads/master-hbase1.x
Commit: 3d38f042eba4f1f1630e70d7724d14f7be5f2ff0
Parents: 792d4ee
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 08:56:35 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 461 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3d38f042/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/3d38f042/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3d38f042/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

kylin git commit: minor, enhance dev-support scripts

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 937cc546d -> 792d4ee3f


minor, enhance dev-support scripts


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/792d4ee3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/792d4ee3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/792d4ee3

Branch: refs/heads/master
Commit: 792d4ee3fd3f12b63a34a47bb90ed7b87449f15a
Parents: 937cc54
Author: Yang Li 
Authored: Tue Oct 4 16:55:46 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 16:55:46 2016 +0800

--
 dev-support/sync_hbase_cdh_branches.cmd | 43 --
 dev-support/sync_hbase_cdh_branches.sh  | 45 
 2 files changed, 45 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/792d4ee3/dev-support/sync_hbase_cdh_branches.cmd
--
diff --git a/dev-support/sync_hbase_cdh_branches.cmd 
b/dev-support/sync_hbase_cdh_branches.cmd
deleted file mode 100644
index ed828ca..000
--- a/dev-support/sync_hbase_cdh_branches.cmd
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/bash
-
-# 
-
-base=master
-
-# 
-
-set -o pipefail  # trace ERR through pipes
-set -o errtrace  # trace ERR through 'time command' and other functions
-function error() {
-   SCRIPT="$0"   # script name
-   LASTLINE="$1" # line of error occurrence
-   LASTERR="$2"  # error code
-   echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code 
${LASTERR}"
-   exit 1
-}
-trap 'error ${LINENO} ${?}' ERR
-
-# 
-
-git fetch apache
-git checkout apache/$base-hbase1.x
-git format-patch -1
-git checkout apache/$base-cdh1.x
-git format-patch -1
-
-git checkout apache/$base
-git checkout -b tmp
-git reset apache/$base --hard
-
-git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-#git push apache tmp:$base-hbase1.x -f
-rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-
-git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
-#git push apache tmp:$base-cdh5.7 -f
-rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
-
-# clean up
-git checkout master
-git reset apache/master --hard
-git checkout -b tmp

http://git-wip-us.apache.org/repos/asf/kylin/blob/792d4ee3/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh 
b/dev-support/sync_hbase_cdh_branches.sh
new file mode 100644
index 000..f52284a
--- /dev/null
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# 
+
+base=master
+
+# 
+
+set -o pipefail  # trace ERR through pipes
+set -o errtrace  # trace ERR through 'time command' and other functions
+function error() {
+   SCRIPT="$0"   # script name
+   LASTLINE="$1" # line of error occurrence
+   LASTERR="$2"  # error code
+   echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code 
${LASTERR}"
+   exit 1
+}
+trap 'error ${LINENO} ${?}' ERR
+
+# 
+
+git fetch apache
+git checkout apache/$base-hbase1.x
+git format-patch -1
+git checkout apache/$base-cdh5.7
+git format-patch -1
+
+git checkout apache/$base
+git checkout -b tmp
+git reset apache/$base --hard
+
+git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
+mvn clean compile -DskipTests
+git push apache tmp:$base-hbase1.x -f
+rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
+
+git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+mvn clean compile -DskipTests
+git push apache tmp:$base-cdh5.7 -f
+rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+
+# clean up
+git checkout master
+git reset apache/master --hard
+git branch -D tmp

[05/31] kylin git commit: Revert "Revert "KYLIN-1818 change kafka dependency to provided""

[liyang]

Revert "Revert "KYLIN-1818 change kafka dependency to provided""

This reverts commit dee8f2d073d225a3b8189c66170fd310c1a8d221.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c738f0f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c738f0f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c738f0f2

Branch: refs/heads/master-cdh5.7
Commit: c738f0f265737dd5849f9342fd694e3cef3344b8
Parents: f89e35f
Author: shaofengshi 
Authored: Sat Sep 24 14:56:32 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 build/bin/find-kafka-dependency.sh  | 12 ++--
 build/bin/kylin.sh  |  2 ++
 .../kylin/engine/mr/common/AbstractHadoopJob.java   |  4 ++--
 3 files changed, 10 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c738f0f2/build/bin/find-kafka-dependency.sh
--
diff --git a/build/bin/find-kafka-dependency.sh 
b/build/bin/find-kafka-dependency.sh
index c6b9c24..7349360 100644
--- a/build/bin/find-kafka-dependency.sh
+++ b/build/bin/find-kafka-dependency.sh
@@ -32,20 +32,20 @@ then
 fi
 
 # works for kafka 9+
-kafka_client=`find -L "$(dirname $kafka_home)" -name 
'kafka-clients-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name 
'*sources*' ''-printf '%p:' | sed 's/:$//'`
-if [ -z "$kafka_client" ]
+kafka_dependency=`find -L $kafka_home -name 
'kafka-clients-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name 
'*sources*' ''-printf '%p:' | sed 's/:$//'`
+if [ -z "$kafka_dependency" ]
 then
 # works for kafka 8
-kafka_broker=`find -L "$(dirname $kafka_home)" -name 
'kafka_[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name 
'*sources*' ''-printf '%p:' | sed 's/:$//'`
-if [ -z "$kafka_broker" ]
+kafka_dependency=`find -L $kafka_home -name 'kafka_[a-z0-9A-Z\.-]*.jar' ! 
-name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 
's/:$//'`
+if [ -z "$kafka_dependency" ]
 then
 echo "kafka client lib not found"
 exit 1
 else
-echo "kafka dependency: $kafka_broker"
+echo "kafka dependency: $kafka_dependency"
 export kafka_dependency
 fi
 else
-echo "kafka dependency: $kafka_client"
+echo "kafka dependency: $kafka_dependency"
 export kafka_dependency
 fi

http://git-wip-us.apache.org/repos/asf/kylin/blob/c738f0f2/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 201b5b6..7a9d2a1 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -31,6 +31,7 @@ function retrieveDependency() {
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh
 source ${dir}/find-hbase-dependency.sh
+source ${dir}/find-kafka-dependency.sh
 
 #retrive $KYLIN_EXTRA_START_OPTS
 if [ -f "${dir}/setenv.sh" ]
@@ -106,6 +107,7 @@ then
 -Djava.io.tmpdir=${tomcat_root}/temp  \
 -Dkylin.hive.dependency=${hive_dependency} \
 -Dkylin.hbase.dependency=${hbase_dependency} \
+-Dkylin.kafka.dependency=${kafka_dependency} \
 -Dkylin.rest.address=${kylin_rest_address} \
 -Dspring.profiles.active=${spring_profile} \
 org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar  
org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.out 
2>&1 & echo $! > ${KYLIN_HOME}/pid &

http://git-wip-us.apache.org/repos/asf/kylin/blob/c738f0f2/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index af2ed9f..a138eec 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -226,11 +226,11 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 }
 }
 
-// for hive dependencies
+// for kafka dependencies
 if (kylinKafkaDependency != null) {
 kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
 
-logger.info("Kafka Dependencies Before Filtered: " + 
kylinHiveDependency);
+logger.info("Kafka Dependencies Before Filtered: " + 
kylinKafkaDependency);
 
 if (kylinDependency.length() > 0)
 kylinDependency.append(",");

[18/31] kylin git commit: KYLIN-2050: Better close resource for AutoCloseable object

[liyang]

KYLIN-2050: Better close resource for AutoCloseable object


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/31b80905
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/31b80905
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/31b80905

Branch: refs/heads/master-cdh5.7
Commit: 31b80905ec3645ecf2c0e8bb5c02bdc451974648
Parents: d366dc1
Author: Yiming Liu 
Authored: Sun Sep 25 21:01:33 2016 +0800
Committer: Li Yang 
Committed: Wed Sep 28 13:37:26 2016 +0800

--
 .../org/apache/kylin/common/util/DBUtils.java   | 89 
 .../apache/kylin/common/util/ZipFileUtils.java  |  3 +-
 .../apache/kylin/dict/lookup/LookupTable.java   |  3 +-
 .../apache/kylin/metadata/MetadataManager.java  |  4 +-
 .../java/org/apache/kylin/query/QueryCli.java   | 13 +--
 .../kylin/query/enumerator/HiveEnumerator.java  | 19 +
 .../apache/kylin/rest/service/CubeService.java  |  5 +-
 .../apache/kylin/rest/service/QueryService.java | 23 +
 .../storage/hbase/steps/CubeHTableUtil.java |  7 +-
 .../storage/hbase/steps/DeprecatedGCStep.java   |  8 +-
 .../hbase/util/HBaseRegionSizeCalculator.java   |  3 +-
 .../storage/hbase/util/HbaseStreamingInput.java |  5 +-
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 13 +--
 13 files changed, 123 insertions(+), 72 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/31b80905/core-common/src/main/java/org/apache/kylin/common/util/DBUtils.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/DBUtils.java 
b/core-common/src/main/java/org/apache/kylin/common/util/DBUtils.java
new file mode 100644
index 000..9ecd3c3
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DBUtils.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DBUtils {
+
+private static final Logger logger = 
LoggerFactory.getLogger(DBUtils.class);
+
+/**
+ * Closes an ResultSet unconditionally.
+ * 
+ * Equivalent to {@link ResultSet#close()}, except any exceptions will be 
ignored.
+ * This is typically used in finally blocks.
+ * 
+ *
+ * @param output the ResultSet to close, may be null or already closed
+ */
+public static void closeQuietly(final ResultSet rs) {
+closeQuietly((AutoCloseable) rs);
+}
+
+/**
+ * Closes an Statement unconditionally.
+ * 
+ * Equivalent to {@link Statement#close()}, except any exceptions will be 
ignored.
+ * This is typically used in finally blocks.
+ * 
+ *
+ * @param output the ResultSet to close, may be null or already closed
+ */
+public static void closeQuietly(final Statement stmt) {
+closeQuietly((AutoCloseable) stmt);
+}
+
+/**
+ * Closes an Connection unconditionally.
+ * 
+ * Equivalent to {@link Connection#close()}, except any exceptions will be 
ignored.
+ * This is typically used in finally blocks.
+ * 
+ *
+ * @param output the ResultSet to close, may be null or already closed
+ */
+public static void closeQuietly(final Connection conn) {
+closeQuietly((AutoCloseable) conn);
+}
+
+/**
+ * Closes a AutoCloseable unconditionally.
+ * 
+ * Equivalent to {@link AutoCloseable#close()}, except any exceptions will 
be ignored. This is typically used in
+ * finally blocks.
+ * 
+ *
+ * @param closeable the objects to close, may be null or already closed
+ */
+public static void closeQuietly(final AutoCloseable closeable) {
+try {
+if (closeable != null) {
+closeable.close();
+}
+} catch (final Exception ioe) {
+logger.debug("", 

[31/31] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59beaa0e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59beaa0e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59beaa0e

Branch: refs/heads/master-cdh5.7
Commit: 59beaa0ed50aece75af63afc46326079bd3ee395
Parents: cad8d91
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Yang Li 
Committed: Tue Oct 4 08:35:06 2016 +

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/59beaa0e/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/59beaa0e/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/59beaa0e/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[29/31] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/cad8d91c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/cad8d91c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[09/31] kylin git commit: KYLIN-1762 fix query test error

[liyang]

KYLIN-1762 fix query test error

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab5563a8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab5563a8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab5563a8

Branch: refs/heads/master-cdh5.7
Commit: ab5563a8ec060fba48ec8f43244bed6f887b0e83
Parents: be18158
Author: shaofengshi 
Authored: Sun Sep 25 21:41:37 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../kylin/job/streaming/Kafka10DataLoader.java  |  2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |  2 +-
 .../mr/steps/FactDistinctColumnPartitioner.java |  3 ++
 .../engine/mr/steps/FactDistinctColumnsJob.java |  2 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 38 +-
 .../mr/steps/FactDistinctHiveColumnsMapper.java | 49 --
 .../kafka/DEFAULT.STREAMING_TABLE.json  |  1 +
 .../kylin/provision/BuildCubeWithStream.java| 52 
 .../kylin/provision/BuildCubeWithStream2.java   |  4 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  3 ++
 .../org/apache/kylin/query/KylinTestBase.java   |  2 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java |  2 +-
 .../source/kafka/TimedJsonStreamParser.java |  7 +--
 13 files changed, 117 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ab5563a8/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java
--
diff --git 
a/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java 
b/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java
index 2b299cc..8c548be 100644
--- 
a/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java
+++ 
b/assembly/src/test/java/org/apache/kylin/job/streaming/Kafka10DataLoader.java
@@ -65,7 +65,7 @@ public class Kafka10DataLoader extends StreamDataLoader {
 props.put("retry.backoff.ms", "1000");
 KafkaProducer producer = KafkaClient.getKafkaProducer(brokerList, 
props);
 
-for (int i = 0; i < messages.size(); ++i) {
+for (int i = 0; i < messages.size(); i++) {
 ProducerRecord keyedMessage = new 
ProducerRecord(clusterConfig.getTopic(), String.valueOf(i), 
messages.get(i));
 producer.send(keyedMessage);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab5563a8/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index 6ca89c8..5cd4f1d 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -143,7 +143,7 @@ public class CubeJoinedFlatTableDesc implements 
IJoinedFlatTableDesc {
 public int getColumnIndex(TblColRef colRef) {
 Integer index = columnIndexMap.get(colRef);
 if (index == null)
-throw new IllegalArgumentException("Column " + colRef.toString() + 
" wasn't found on flat table.");
+return -1;
 
 return index.intValue();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ab5563a8/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
index a631cf4..6973c4b 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnPartitioner.java
@@ -34,6 +34,9 @@ public class FactDistinctColumnPartitioner extends 
Partitioner {
 if (key.getBytes()[0] == FactDistinctHiveColumnsMapper.MARK_FOR_HLL) {
 // the last reducer is for merging hll
 return numReduceTasks - 1;
+} else if (key.getBytes()[0] == 
FactDistinctHiveColumnsMapper.MARK_FOR_PARTITION_COL) {
+// the last reducer is for merging hll
+return numReduceTasks - 2;
 } else {
 int colIndex = BytesUtil.readUnsigned(key.getBytes(), 0, 1);
 return colIndex;

[30/31] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cad8d91c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cad8d91c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cad8d91c

Branch: refs/heads/master-cdh5.7
Commit: cad8d91c3ad23eb0b53788ceed59a8379b79cea6
Parents: 937cc54
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 08:31:03 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 461 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cad8d91c/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF 

[17/31] kylin git commit: KYLIN-2035 only auto-merge after a READY segment is built

[liyang]

KYLIN-2035 only auto-merge after a READY segment is built


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d366dc11
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d366dc11
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d366dc11

Branch: refs/heads/master-cdh5.7
Commit: d366dc1118aeb342a2ace251bc6dfac976f00e14
Parents: 859230d
Author: Li Yang 
Authored: Wed Sep 28 11:51:50 2016 +0800
Committer: Li Yang 
Committed: Wed Sep 28 11:51:50 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeInstance.java  | 12 
 .../org/apache/kylin/rest/service/CubeService.java | 13 -
 2 files changed, 20 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d366dc11/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index cad00e1..720690d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -293,6 +293,18 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return latest;
 }
 
+public CubeSegment getLatestBuiltSegment() {
+CubeSegment latest = null;
+for (int i = segments.size() - 1; i >= 0; i--) {
+CubeSegment seg = segments.get(i);
+if (seg.getLastBuildTime() > 0) {
+if (latest == null || seg.getLastBuildTime() > 
latest.getLastBuildTime())
+latest = seg;
+}
+}
+return latest;
+}
+
 public List getSegments() {
 return segments;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d366dc11/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index e446045..4fa07b6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -582,15 +582,18 @@ public class CubeService extends BasicService {
 }
 
 public void updateOnNewSegmentReady(String cubeName) {
-logger.debug("on updateOnNewSegmentReady: " + cubeName);
 final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 String serverMode = kylinConfig.getServerMode();
-logger.debug("server mode: " + serverMode);
 if (Constant.SERVER_MODE_JOB.equals(serverMode.toLowerCase()) || 
Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase())) {
-keepCubeRetention(cubeName);
-mergeCubeSegment(cubeName);
+CubeInstance cube = getCubeManager().getCube(cubeName);
+if (cube != null) {
+CubeSegment seg = cube.getLatestBuiltSegment();
+if (seg != null && seg.getStatus() == SegmentStatusEnum.READY) 
{
+keepCubeRetention(cubeName);
+mergeCubeSegment(cubeName);
+}
+}
 }
-
 }
 
 private void keepCubeRetention(String cubeName) {

[23/31] kylin git commit: minor, retry when http failed in job diagnosis

[liyang]

minor, retry when http failed in job diagnosis


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e049817
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e049817
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e049817

Branch: refs/heads/master-cdh5.7
Commit: 1e049817856ede06c7c8736ad1d608765f301a21
Parents: 51b5b31
Author: lidongsjtu 
Authored: Thu Sep 29 09:48:58 2016 +0800
Committer: lidongsjtu 
Committed: Thu Sep 29 15:34:25 2016 +0800

--
 .../java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java | 4 ++--
 .../java/org/apache/kylin/tool/JobTaskCounterExtractor.java | 9 +++--
 2 files changed, 9 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e049817/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java 
b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
index 635a2c3..c8ff1f4 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
@@ -99,8 +99,8 @@ public class JobDiagnosisInfoCLI extends 
AbstractInfoExtractor {
 addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + jobId);
 addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + jobId);
 for (ExecutablePO task : executablePO.getTasks()) {
-addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + 
executablePO.getTasks());
-addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + 
executablePO.getTasks());
+addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + 
task.getUuid());
+addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + 
task.getUuid());
 if (includeYarnLogs) {
 yarnLogsResources.add(task.getUuid());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e049817/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
--
diff --git 
a/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java 
b/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
index 7b9d8f3..6a317e9 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
@@ -47,6 +47,8 @@ public class JobTaskCounterExtractor extends 
AbstractInfoExtractor {
 private String yarnUrl;
 private static final Logger logger = 
LoggerFactory.getLogger(JobTaskCounterExtractor.class);
 
+private final int HTTP_RETRY = 3;
+
 public JobTaskCounterExtractor(String mrJobId) {
 this.mrJobId = mrJobId;
 this.yarnUrl = getRestCheckUrl();
@@ -87,14 +89,17 @@ public class JobTaskCounterExtractor extends 
AbstractInfoExtractor {
 private String getHttpResponse(String url) {
 HttpClient client = new HttpClient();
 String response = null;
-while (response == null) {
+int retry_times = 0;
+while (response == null && retry_times < HTTP_RETRY) {
+retry_times++;
+
 HttpMethod get = new GetMethod(url);
 try {
 get.addRequestHeader("accept", "application/json");
 client.executeMethod(get);
 response = get.getResponseBodyAsString();
 } catch (Exception e) {
-logger.warn("Failed to fetch http response" + e);
+logger.warn("Failed to fetch http response. Retry={}", 
retry_times, e);
 } finally {
 get.releaseConnection();
 }

[07/31] kylin git commit: Revert "Revert "KYLIN-1726 Scalable streaming cubing""

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/8431af45/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
new file mode 100644
index 000..decfb60
--- /dev/null
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/hadoop/KafkaFlatTableJob.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.kafka.hadoop;
+
+import org.apache.kylin.source.kafka.util.KafkaClient;
+import org.apache.kylin.source.kafka.util.KafkaOffsetMapping;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.source.kafka.KafkaConfigManager;
+import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Run a Hadoop Job to process the stream data in kafka;
+ * Modified from the kafka-hadoop-loader in 
https://github.com/amient/kafka-hadoop-loader
+ */
+public class KafkaFlatTableJob extends AbstractHadoopJob {
+protected static final Logger logger = 
LoggerFactory.getLogger(KafkaFlatTableJob.class);
+
+public static final String CONFIG_KAFKA_PARITION_MIN = 
"kafka.partition.min";
+public static final String CONFIG_KAFKA_PARITION_MAX = 
"kafka.partition.max";
+public static final String CONFIG_KAFKA_PARITION_START = 
"kafka.partition.start.";
+public static final String CONFIG_KAFKA_PARITION_END = 
"kafka.partition.end.";
+
+public static final String CONFIG_KAFKA_BROKERS = "kafka.brokers";
+public static final String CONFIG_KAFKA_TOPIC = "kafka.topic";
+public static final String CONFIG_KAFKA_TIMEOUT = "kafka.connect.timeout";
+public static final String CONFIG_KAFKA_BUFFER_SIZE = 
"kafka.connect.buffer.size";
+public static final String CONFIG_KAFKA_CONSUMER_GROUP = 
"kafka.consumer.group";
+public static final String CONFIG_KAFKA_INPUT_FORMAT = "input.format";
+public static final String CONFIG_KAFKA_PARSER_NAME = "kafka.parser.name";
+@Override
+public int run(String[] args) throws Exception {
+Options options = new Options();
+
+try {
+options.addOption(OPTION_JOB_NAME);
+options.addOption(OPTION_CUBE_NAME);
+options.addOption(OPTION_OUTPUT_PATH);
+options.addOption(OPTION_SEGMENT_NAME);
+parseOptions(options, args);
+
+job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
+String cubeName = getOptionValue(OPTION_CUBE_NAME);
+Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
+
+String segmentName = getOptionValue(OPTION_SEGMENT_NAME);
+
+// 

+// add metadata to distributed cache
+CubeManager cubeMgr = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+CubeInstance cube = cubeMgr.getCube(cubeName);
+
+job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, 
segmentName);
+logger.info("Starting: " + job.getJobName());
+
+setJobClasspath(job, cube.getConfig());
+
+  

[15/31] kylin git commit: Revert "Revert "stop scheduler on test finish""

[liyang]

Revert "Revert "stop scheduler on test finish""

This reverts commit e604f6527d60b767f8a46a576765cfd205b1efc5.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8cbffb40
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8cbffb40
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8cbffb40

Branch: refs/heads/master-cdh5.7
Commit: 8cbffb40477611aa9c06bc5c22787aef83bc28d3
Parents: b1a0f4d
Author: shaofengshi 
Authored: Sat Sep 24 14:58:26 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../apache/kylin/job/impl/threadpool/BaseSchedulerTest.java| 1 +
 .../java/org/apache/kylin/provision/BuildCubeWithEngine.java   | 6 ++
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java   | 1 +
 .../java/org/apache/kylin/provision/BuildCubeWithStream2.java  | 1 +
 4 files changed, 9 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbffb40/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index 4e092a1..97c9f8d 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -55,6 +55,7 @@ public abstract class BaseSchedulerTest extends 
LocalFileMetadataTestCase {
 
 @After
 public void after() throws Exception {
+DefaultScheduler.destroyInstance();
 cleanupTestMetadata();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbffb40/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 3d60a3c..31cf0eb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -79,6 +79,7 @@ public class BuildCubeWithEngine {
 BuildCubeWithEngine buildCubeWithEngine = new 
BuildCubeWithEngine();
 buildCubeWithEngine.before();
 buildCubeWithEngine.build();
+buildCubeWithEngine.after();
 logger.info("Build is done");
 afterClass();
 logger.info("Going to exit");
@@ -148,6 +149,11 @@ public class BuildCubeWithEngine {
 
 }
 
+
+public void after(){
+DefaultScheduler.destroyInstance();
+}
+
 public static void afterClass() {
 HBaseMetadataTestCase.staticCleanupTestMetadata();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbffb40/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index b7c609e..6e5313f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -217,6 +217,7 @@ public class BuildCubeWithStream {
 
 public void after() {
 kafkaServer.stop();
+DefaultScheduler.destroyInstance();
 }
 
 protected void waitForJob(String jobId) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbffb40/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
index d48a473..2812446 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
@@ -235,6 +235,7 @@ public class BuildCubeWithStream2 {
 
 public void after() {
 kafkaServer.stop();
+DefaultScheduler.destroyInstance();
 }
 
 protected void waitForJob(String jobId) {

[22/31] kylin git commit: KYLIN-1962: Revert unsed split config feature

[liyang]

KYLIN-1962: Revert unsed split config feature

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/51b5b31b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/51b5b31b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/51b5b31b

Branch: refs/heads/master-cdh5.7
Commit: 51b5b31bbce3aac1d3a3c0de7746334961d5ffcd
Parents: 0fd4ad5
Author: Yiming Liu 
Authored: Tue Sep 27 07:52:52 2016 +0800
Committer: shaofengshi 
Committed: Wed Sep 28 17:51:49 2016 +0800

--
 .../org/apache/kylin/common/KylinConfig.java| 45 ---
 .../apache/kylin/common/KylinConfigTest.java| 12 
 .../test_case_data/localmeta/kylin.properties   | 33 ++-
 .../localmeta/kylin_account.properties  | 59 
 .../org/apache/kylin/tool/DiagnosisInfoCLI.java |  4 +-
 5 files changed, 33 insertions(+), 120 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/51b5b31b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index f134ad4..9543fc7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -47,7 +47,6 @@ public class KylinConfig extends KylinConfigBase {
 
 /** Kylin properties file name */
 public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties";
-public static final String KYLIN_ACCOUNT_CONF_PROPERTIES_FILE = 
"kylin_account.properties";
 public static final String KYLIN_CONF = "KYLIN_CONF";
 
 // static cached instances
@@ -205,23 +204,6 @@ public class KylinConfig extends KylinConfigBase {
 return getKylinPropertiesFile(path);
 }
 
-static File getKylinAccountPropertiesFile() {
-String kylinConfHome = System.getProperty(KYLIN_CONF);
-if (!StringUtils.isEmpty(kylinConfHome)) {
-logger.info("Use KYLIN_CONF=" + kylinConfHome);
-return getKylinAccountPropertiesFile(kylinConfHome);
-}
-
-logger.warn("KYLIN_CONF property was not set, will seek KYLIN_HOME env 
variable");
-
-String kylinHome = getKylinHome();
-if (StringUtils.isEmpty(kylinHome))
-throw new KylinConfigCannotInitException("Didn't find KYLIN_CONF 
or KYLIN_HOME, please set one of them");
-
-String path = kylinHome + File.separator + "conf";
-return getKylinAccountPropertiesFile(path);
-}
-
 public static Properties getKylinProperties() {
 File propFile = getKylinPropertiesFile();
 if (propFile == null || !propFile.exists()) {
@@ -242,25 +224,6 @@ public class KylinConfig extends KylinConfigBase {
 IOUtils.closeQuietly(ois);
 conf.putAll(propOverride);
 }
-
-File accountPropFile = getKylinAccountPropertiesFile();
-if (accountPropFile.exists()) {
-FileInputStream ois = new FileInputStream(accountPropFile);
-Properties propAccount = new Properties();
-propAccount.load(ois);
-IOUtils.closeQuietly(ois);
-conf.putAll(propAccount);
-}
-
-File accountPropOverrideFile = new 
File(accountPropFile.getParentFile(), accountPropFile.getName() + ".override");
-if (accountPropOverrideFile.exists()) {
-FileInputStream ois = new 
FileInputStream(accountPropOverrideFile);
-Properties propAccountOverride = new Properties();
-propAccountOverride.load(ois);
-IOUtils.closeQuietly(ois);
-conf.putAll(propAccountOverride);
-}
-
 } catch (IOException e) {
 throw new RuntimeException(e);
 }
@@ -282,14 +245,6 @@ public class KylinConfig extends KylinConfigBase {
 return new File(path, KYLIN_CONF_PROPERTIES_FILE);
 }
 
-private static File getKylinAccountPropertiesFile(String path) {
-if (path == null) {
-return null;
-}
-
-return new File(path, KYLIN_ACCOUNT_CONF_PROPERTIES_FILE);
-}
-
 public static void setSandboxEnvIfPossible() {
 File dir1 = new File("../examples/test_case_data/sandbox");
 File dir2 = new File("../../kylin/examples/test_case_data/sandbox");

http://git-wip-us.apache.org/repos/asf/kylin/blob/51b5b31b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java

[08/31] kylin git commit: Revert "Revert "KYLIN-1726 Scalable streaming cubing""

[liyang]

Revert "Revert "KYLIN-1726 Scalable streaming cubing""

This reverts commit 506cd783132023a06f1669ad248b74bf9d96d0e1.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8431af45
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8431af45
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8431af45

Branch: refs/heads/master-cdh5.7
Commit: 8431af45528abb2d39a69b5e762712983573e5a6
Parents: a00d1e3
Author: shaofengshi 
Authored: Sat Sep 24 14:55:59 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  35 ++-
 .../kylin/job/streaming/Kafka10DataLoader.java  |  80 +++
 .../apache/kylin/common/KylinConfigBase.java|   1 +
 .../java/org/apache/kylin/cube/CubeSegment.java |   1 +
 .../java/org/apache/kylin/cube/ISegment.java|  39 
 .../cube/gridtable/SegmentGTStartAndEnd.java|   2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |   6 +
 .../cube/model/CubeJoinedFlatTableEnrich.java   |   6 +
 .../apache/kylin/gridtable/ScannerWorker.java   |   2 +-
 .../metadata/model/IJoinedFlatTableDesc.java|   2 +
 .../apache/kylin/metadata/model/ISegment.java   |  36 +++
 .../kylin/engine/mr/BatchMergeJobBuilder2.java  |   3 +
 .../org/apache/kylin/engine/mr/IMRInput.java|  10 +
 .../java/org/apache/kylin/engine/mr/MRUtil.java |   4 +
 .../test_streaming_table_model_desc.json|   6 +-
 .../kylin/provision/BuildCubeWithStream.java| 218 +-
 .../org/apache/kylin/provision/MockKafka.java   | 191 
 .../apache/kylin/provision/NetworkUtils.java|  52 +
 pom.xml |   2 +-
 .../apache/kylin/source/hive/HiveMRInput.java   |  11 +
 source-kafka/pom.xml|  13 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java | 221 +++
 .../apache/kylin/source/kafka/KafkaSource.java  |  57 +
 .../kylin/source/kafka/KafkaStreamingInput.java |  17 +-
 .../kylin/source/kafka/MergeOffsetStep.java |  89 
 .../kylin/source/kafka/SeekOffsetStep.java  | 119 ++
 .../kylin/source/kafka/UpdateTimeRangeStep.java | 108 +
 .../source/kafka/config/KafkaClusterConfig.java |   3 +-
 .../source/kafka/hadoop/KafkaFlatTableJob.java  | 165 ++
 .../kafka/hadoop/KafkaFlatTableMapper.java  |  51 +
 .../source/kafka/hadoop/KafkaInputFormat.java   |  98 
 .../kafka/hadoop/KafkaInputRecordReader.java| 166 ++
 .../source/kafka/hadoop/KafkaInputSplit.java| 102 +
 .../kylin/source/kafka/util/KafkaClient.java| 115 ++
 .../source/kafka/util/KafkaOffsetMapping.java   |  97 
 .../kylin/source/kafka/util/KafkaRequester.java |   7 +-
 .../kylin/source/kafka/util/KafkaUtils.java |   3 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |   2 +-
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |   2 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |   2 +-
 40 files changed, 2024 insertions(+), 120 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8431af45/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 8c64f91..9b282e3 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -143,14 +143,12 @@ public class DeployUtil {
 deployHiveTables();
 }
 
-public static void prepareTestDataForStreamingCube(long startTime, long 
endTime, String cubeName, StreamDataLoader streamDataLoader) throws IOException 
{
+public static void prepareTestDataForStreamingCube(long startTime, long 
endTime, int numberOfRecords, String cubeName, StreamDataLoader 
streamDataLoader) throws IOException {
 CubeInstance cubeInstance = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
-List data = StreamingTableDataGenerator.generate(1, 
startTime, endTime, cubeInstance.getFactTable());
-List data2 = StreamingTableDataGenerator.generate(10, endTime, 
endTime + 30, cubeInstance.getFactTable());
+List data = 
StreamingTableDataGenerator.generate(numberOfRecords, startTime, endTime, 
cubeInstance.getFactTable());
 TableDesc tableDesc = cubeInstance.getFactTableDesc();
 //load into kafka
 streamDataLoader.loadIntoKafka(data);
-streamDataLoader.loadIntoKafka(data2);
 logger.info("Write {} messages into {}", data.size(), 
streamDataLoader.toString());
 
 //csv 

[21/31] kylin git commit: minor add a blank line in create_sample_table.sql for beeline

[liyang]

minor add a blank line in create_sample_table.sql for beeline


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fd4ad58
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fd4ad58
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fd4ad58

Branch: refs/heads/master-cdh5.7
Commit: 0fd4ad587e3710324eaf58266b64f202eeabfa7e
Parents: e500726
Author: shaofengshi 
Authored: Wed Sep 28 17:51:36 2016 +0800
Committer: shaofengshi 
Committed: Wed Sep 28 17:51:36 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd4ad58/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index d83566c..5ed7b9b 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -187,4 +187,4 @@ STORED AS TEXTFILE;
 
 LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' 
OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES;
 LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' 
OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT;
-LOAD DATA INPATH 
'/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE 
INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;
\ No newline at end of file
+LOAD DATA INPATH 
'/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE 
INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;

[12/31] kylin git commit: Revert "Revert "KYLIN-1762 discard job when no stream message""

[liyang]

Revert "Revert "KYLIN-1762 discard job when no stream message""

This reverts commit da5ba276b972f8b3c0d220252e74ac2ff73298fc.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/25f8ffc0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/25f8ffc0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/25f8ffc0

Branch: refs/heads/master-cdh5.7
Commit: 25f8ffc0ee16b1702723d5530b22084a608fed9a
Parents: ae3d7e4
Author: shaofengshi 
Authored: Sat Sep 24 14:57:01 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../job/execution/DefaultChainedExecutable.java |  6 +++
 .../kylin/source/kafka/SeekOffsetStep.java  | 45 +++-
 2 files changed, 41 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/25f8ffc0/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 753b389..39a5f4f 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -88,6 +88,7 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 boolean allSucceed = true;
 boolean hasError = false;
 boolean hasRunning = false;
+boolean hasDiscarded = false;
 for (Executable task : jobs) {
 final ExecutableState status = task.getStatus();
 if (status == ExecutableState.ERROR) {
@@ -99,6 +100,9 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 if (status == ExecutableState.RUNNING) {
 hasRunning = true;
 }
+if (status == ExecutableState.DISCARDED) {
+hasDiscarded = true;
+}
 }
 if (allSucceed) {
 setEndTime(System.currentTimeMillis());
@@ -110,6 +114,8 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 notifyUserStatusChange(executableContext, 
ExecutableState.ERROR);
 } else if (hasRunning) {
 jobService.updateJobOutput(getId(), ExecutableState.RUNNING, 
null, null);
+} else if (hasDiscarded) {
+jobService.updateJobOutput(getId(), ExecutableState.DISCARDED, 
null, null);
 } else {
 jobService.updateJobOutput(getId(), ExecutableState.READY, 
null, null);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/25f8ffc0/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
index 5dca93f..479f1b8 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
@@ -17,6 +17,10 @@
 */
 package org.apache.kylin.source.kafka;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Maps;
+import org.apache.commons.math3.util.MathUtils;
 import org.apache.kylin.source.kafka.util.KafkaClient;
 import org.apache.kylin.source.kafka.util.KafkaOffsetMapping;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -34,6 +38,7 @@ import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
@@ -101,19 +106,39 @@ public class SeekOffsetStep extends AbstractExecutable {
 }
 }
 
-KafkaOffsetMapping.saveOffsetStart(segment, startOffsets);
-KafkaOffsetMapping.saveOffsetEnd(segment, endOffsets);
+long totalStartOffset = 0, totalEndOffset = 0;
+for (Long v : startOffsets.values()) {
+totalStartOffset += v;
+}
+for (Long v : endOffsets.values()) {
+totalEndOffset += v;
+}
 
-segment.setName(CubeSegment.makeSegmentName(0, 0, 
segment.getSourceOffsetStart(), segment.getSourceOffsetEnd()));
-CubeUpdate cubeBuilder = new CubeUpdate(cube);
-

[10/31] kylin git commit: Revert "Revert "KYLIN-1726 allow job discard itself""

[liyang]

Revert "Revert "KYLIN-1726 allow job discard itself""

This reverts commit 3ae2549ba89e3a2c8ed94a2089678227cf78312d.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0de0239
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0de0239
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0de0239

Branch: refs/heads/master-cdh5.7
Commit: f0de02392f4808e08432417cb00f2ab5fa829055
Parents: de2f4e2
Author: shaofengshi 
Authored: Sat Sep 24 14:57:36 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 14 ++-
 .../kylin/job/execution/AbstractExecutable.java |  2 +
 .../job/execution/DefaultChainedExecutable.java |  2 +
 .../kylin/job/execution/ExecuteResult.java  |  4 ++
 .../kylin/job/DiscardedTestExecutable.java  | 41 
 .../impl/threadpool/DefaultSchedulerTest.java   | 16 
 6 files changed, 68 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f0de0239/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 5a4b07c..3a327f9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -357,34 +357,26 @@ public class CubeManager implements IRealizationProvider {
 Iterator iterator = newSegs.iterator();
 while (iterator.hasNext()) {
 CubeSegment currentSeg = iterator.next();
-boolean found = false;
 for (CubeSegment toRemoveSeg : update.getToRemoveSegs()) {
 if (currentSeg.getUuid().equals(toRemoveSeg.getUuid())) {
+logger.info("Remove segment " + currentSeg.toString());
+
toRemoveResources.add(currentSeg.getStatisticsResourcePath());
 iterator.remove();
-
toRemoveResources.add(toRemoveSeg.getStatisticsResourcePath());
-found = true;
+break;
 }
 }
-if (found == false) {
-logger.error("Segment '" + currentSeg.getName() + "' 
doesn't exist for remove.");
-}
 }
 
 }
 
 if (update.getToUpdateSegs() != null) {
 for (CubeSegment segment : update.getToUpdateSegs()) {
-boolean found = false;
 for (int i = 0; i < newSegs.size(); i++) {
 if (newSegs.get(i).getUuid().equals(segment.getUuid())) {
 newSegs.set(i, segment);
-found = true;
 break;
 }
 }
-if (found == false) {
-logger.error("Segment '" + segment.getName() + "' doesn't 
exist for update.");
-}
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f0de0239/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 90e4d3c..b4ca469 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -74,6 +74,8 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 if (!isDiscarded()) {
 if (result.succeed()) {
 executableManager.updateJobOutput(getId(), 
ExecutableState.SUCCEED, null, result.output());
+} else if (result.discarded()) {
+executableManager.updateJobOutput(getId(), 
ExecutableState.DISCARDED, null, result.output());
 } else {
 executableManager.updateJobOutput(getId(), 
ExecutableState.ERROR, null, result.output());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f0de0239/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 39a5f4f..5a57b05 100644

[28/31] kylin git commit: minor, enhance dev-support scripts

[liyang]

minor, enhance dev-support scripts


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/937cc546
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/937cc546
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/937cc546

Branch: refs/heads/master-cdh5.7
Commit: 937cc546d1dc34832217f1b806f3e91aa486caa4
Parents: e22039d
Author: Yang Li 
Authored: Tue Oct 4 15:45:45 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 15:45:45 2016 +0800

--
 dev-support/sync_hbase_cdh_branches.cmd   | 44 +-
 dev-support/test_all_against_hdp_2_2_4_2_2.sh |  2 +-
 2 files changed, 35 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/937cc546/dev-support/sync_hbase_cdh_branches.cmd
--
diff --git a/dev-support/sync_hbase_cdh_branches.cmd 
b/dev-support/sync_hbase_cdh_branches.cmd
index 0febfbb..ed828ca 100644
--- a/dev-support/sync_hbase_cdh_branches.cmd
+++ b/dev-support/sync_hbase_cdh_branches.cmd
@@ -1,19 +1,43 @@
-git checkout master
-git pull
-git reset apache/master --hard
+#!/bin/bash
+
+# 
+
+base=master
+
+# 
+
+set -o pipefail  # trace ERR through pipes
+set -o errtrace  # trace ERR through 'time command' and other functions
+function error() {
+   SCRIPT="$0"   # script name
+   LASTLINE="$1" # line of error occurrence
+   LASTERR="$2"  # error code
+   echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code 
${LASTERR}"
+   exit 1
+}
+trap 'error ${LINENO} ${?}' ERR
+
+# 
 
-git checkout apache/1.5.x-HBase1.x
+git fetch apache
+git checkout apache/$base-hbase1.x
 git format-patch -1
-git checkout master
+git checkout apache/$base-cdh1.x
+git format-patch -1
+
+git checkout apache/$base
+git checkout -b tmp
+git reset apache/$base --hard
+
 git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-git push apache master:1.5.x-HBase1.x -f
+#git push apache tmp:$base-hbase1.x -f
 rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
 
-git checkout apache/1.5.x-CDH5.7
-git format-patch -1
-git checkout master
 git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
-git push apache master:1.5.x-CDH5.7 -f
+#git push apache tmp:$base-cdh5.7 -f
 rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
 
+# clean up
+git checkout master
 git reset apache/master --hard
+git checkout -b tmp

http://git-wip-us.apache.org/repos/asf/kylin/blob/937cc546/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
index f0d27e4..f7780dd 100644
--- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -22,4 +22,4 @@ cd ${dir}
 cd ..
 
 mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=2.2.4.2-2 -fae 2>&1 | tee mvnverify.log
+mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee 
mvnverify.log

[24/31] kylin git commit: enlarge the job interval in BuildCubeWithStream

[liyang]

enlarge the job interval in BuildCubeWithStream


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dfb41765
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dfb41765
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dfb41765

Branch: refs/heads/master-cdh5.7
Commit: dfb4176531046e212649dce89aebb8d091fc5689
Parents: 1e04981
Author: shaofengshi 
Authored: Fri Sep 30 11:02:09 2016 +0800
Committer: shaofengshi 
Committed: Fri Sep 30 11:02:09 2016 +0800

--
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb41765/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 23d7ca8..b48f75b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -22,9 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Random;
 import java.util.TimeZone;
 import java.util.UUID;
@@ -179,7 +177,7 @@ public class BuildCubeWithStream {
 
 List futures = Lists.newArrayList();
 for (int i = 0; i < 5; i++) {
-Thread.sleep(2 * 60 * 1000); // wait for new messages
+Thread.sleep(5 * 60 * 1000); // wait for new messages
 FutureTask futureTask = new FutureTask(new 
Callable() {
 @Override
 public ExecutableState call() {

[25/31] kylin git commit: fix it, run StorageCleanup after finish BuildCubeWithStream

[liyang]

fix it, run StorageCleanup after finish BuildCubeWithStream

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a6b1f2d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a6b1f2d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a6b1f2d

Branch: refs/heads/master-cdh5.7
Commit: 3a6b1f2d1cd1a42837353e52e3fa219c0d5a314b
Parents: dfb4176
Author: shaofengshi 
Authored: Fri Sep 30 14:04:12 2016 +0800
Committer: shaofengshi 
Committed: Fri Sep 30 14:04:12 2016 +0800

--
 .../org/apache/kylin/provision/BuildCubeWithStream.java  | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a6b1f2d/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index b48f75b..58715f1 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit;
 import com.google.common.collect.Lists;
 import org.I0Itec.zkclient.ZkConnection;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.util.ToolRunner;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
@@ -58,6 +59,7 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.source.kafka.KafkaConfigManager;
 import org.apache.kylin.source.kafka.config.BrokerConfig;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
 import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
 import org.junit.Assert;
 import org.slf4j.Logger;
@@ -177,7 +179,7 @@ public class BuildCubeWithStream {
 
 List futures = Lists.newArrayList();
 for (int i = 0; i < 5; i++) {
-Thread.sleep(5 * 60 * 1000); // wait for new messages
+Thread.sleep(2 * 60 * 1000); // wait for new messages
 FutureTask futureTask = new FutureTask(new 
Callable() {
 @Override
 public ExecutableState call() {
@@ -276,6 +278,7 @@ public class BuildCubeWithStream {
 }
 
 public static void afterClass() throws Exception {
+cleanupOldStorage();
 HBaseMetadataTestCase.staticCleanupTestMetadata();
 }
 
@@ -299,6 +302,12 @@ public class BuildCubeWithStream {
 }
 }
 
+private static void cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+StorageCleanupJob cli = new StorageCleanupJob();
+cli.execute(args);
+}
+
 public static void main(String[] args) throws Exception {
 try {
 beforeClass();

[26/31] kylin git commit: minor, fix sample cube desc

[liyang]

minor, fix sample cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92fc0e53
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92fc0e53
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92fc0e53

Branch: refs/heads/master-cdh5.7
Commit: 92fc0e53845a30a320467e3657a90343d2002e8c
Parents: 3a6b1f2
Author: Yang Li 
Authored: Fri Sep 30 21:46:31 2016 +0800
Committer: Yang Li 
Committed: Fri Sep 30 21:48:35 2016 +0800

--
 .../org/apache/kylin/engine/mr/HadoopUtil.java  |  3 +-
 .../cube_desc/kylin_sales_cube_desc.json| 46 ++--
 2 files changed, 24 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/92fc0e53/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 0ed39d5..88692a0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -34,6 +34,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class HadoopUtil {
+@SuppressWarnings("unused")
 private static final Logger logger = 
LoggerFactory.getLogger(HadoopUtil.class);
 private static final ThreadLocal hadoopConfig = new 
ThreadLocal<>();
 
@@ -45,11 +46,9 @@ public class HadoopUtil {
 if (hadoopConfig.get() == null) {
 Configuration conf = healSickConfig(new Configuration());
 // do not cache this conf, or will affect following mr jobs
-logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 return conf;
 }
 Configuration conf = hadoopConfig.get();
-logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 return conf;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/92fc0e53/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index 70b1294..99f8f46 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -1,7 +1,10 @@
 {
   "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
   "name" : "kylin_sales_cube_desc",
+  "model_name" : "kylin_sales_model",
   "description" : null,
+  "null_string" : null,
   "dimensions" : [ {
 "name" : "CAL_DT",
 "table" : "DEFAULT.KYLIN_CAL_DT",
@@ -106,21 +109,21 @@
 },
 "dependent_measure_ref" : null
   }, {
-"name": "TOP_SELLER",
-"function": {
-  "expression": "TOP_N",
-  "parameter": {
-"type": "column",
-"value": "PRICE",
-"next_parameter": {
-  "type": "column",
-  "value": "SELLER_ID",
-  "next_parameter": null
+"name" : "TOP_SELLER",
+"function" : {
+  "expression" : "TOP_N",
+  "parameter" : {
+"type" : "column",
+"value" : "PRICE",
+"next_parameter" : {
+  "type" : "column",
+  "value" : "SELLER_ID",
+  "next_parameter" : null
 }
   },
-  "returntype": "topn(100)"
+  "returntype" : "topn(100)"
 },
-"dependent_measure_ref": null
+"dependent_measure_ref" : null
   } ],
   "rowkey" : {
 "rowkey_columns" : [ {
@@ -146,21 +149,18 @@
   "encoding" : "dict"
 } ]
   },
-  "last_modified" : 1451468470824,
-  "model_name" : "kylin_sales_model",
-  "null_string" : null,
   "hbase_mapping" : {
 "column_family" : [ {
-  "name" : "f1",
+  "name" : "F1",
   "columns" : [ {
-"qualifier" : "m",
-"measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
+"qualifier" : "M",
+"measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", 
"TOP_SELLER" ]
   } ]
 }, {
-  "name" : "f2",
+  "name" : "F2",
   "columns" : [ {
-"qualifier" : "m",
-"measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT", "TOP_SELLER" 
]
+"qualifier" : "M",
+"measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT" ]
   } ]
 } ]
   },
@@ -174,10 +174,10 @@
   } ],
   "notify_list" : null,
   "status_need_notify" : [ ],
+  "partition_date_start" : 132537600,
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
   "engine_type" : %default_engine_type%,

[14/31] kylin git commit: Revert "Revert "refactor BuildCubeWithStream""

[liyang]

Revert "Revert "refactor BuildCubeWithStream""

This reverts commit 8e9c4550bb562b497442b17eec6485ae96e848d8.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be18158d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be18158d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be18158d

Branch: refs/heads/master-cdh5.7
Commit: be18158dcc5ce739c272b9345d3b2296c3936ee3
Parents: 8cbffb4
Author: shaofengshi 
Authored: Sat Sep 24 14:58:43 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |   7 +-
 .../kylin/provision/BuildCubeWithStream.java|  10 +-
 .../kylin/provision/BuildCubeWithStream2.java   | 145 +--
 3 files changed, 12 insertions(+), 150 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/be18158d/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 9b282e3..9e9df05 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -187,6 +187,7 @@ public class DeployUtil {
 File tmpFile = File.createTempFile(factTableName, "csv");
 FileOutputStream out = new FileOutputStream(tmpFile);
 
+InputStream tempIn = null;
 try {
 if (store.exists(factTablePath)) {
 InputStream oldContent = 
store.getResource(factTablePath).inputStream;
@@ -194,13 +195,15 @@ public class DeployUtil {
 }
 IOUtils.copy(in, out);
 IOUtils.closeQuietly(in);
+IOUtils.closeQuietly(out);
 
 store.deleteResource(factTablePath);
-in = new FileInputStream(tmpFile);
-store.putResource(factTablePath, in, System.currentTimeMillis());
+tempIn = new FileInputStream(tmpFile);
+store.putResource(factTablePath, tempIn, 
System.currentTimeMillis());
 } finally {
 IOUtils.closeQuietly(out);
 IOUtils.closeQuietly(in);
+IOUtils.closeQuietly(tempIn);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/be18158d/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 6e5313f..bfe1d0a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -62,10 +62,10 @@ public class BuildCubeWithStream {
 
 private static final Logger logger = 
LoggerFactory.getLogger(org.apache.kylin.provision.BuildCubeWithStream.class);
 
-private CubeManager cubeManager;
+protected CubeManager cubeManager;
 private DefaultScheduler scheduler;
 protected ExecutableManager jobService;
-private static final String cubeName = "test_streaming_table_cube";
+static final String cubeName = "test_streaming_table_cube";
 
 private KafkaConfig kafkaConfig;
 private MockKafka kafkaServer;
@@ -114,13 +114,13 @@ public class BuildCubeWithStream {
 Assert.assertEquals(topicName, topicMetadata.topic());
 }
 
-private void generateStreamData(long startTime, long endTime, int 
numberOfRecords) throws IOException {
+protected void generateStreamData(long startTime, long endTime, int 
numberOfRecords) throws IOException {
 Kafka10DataLoader dataLoader = new Kafka10DataLoader(kafkaConfig);
 DeployUtil.prepareTestDataForStreamingCube(startTime, endTime, 
numberOfRecords, cubeName, dataLoader);
 logger.info("Test data inserted into Kafka");
 }
 
-private void clearSegment(String cubeName) throws Exception {
+protected void clearSegment(String cubeName) throws Exception {
 CubeInstance cube = cubeManager.getCube(cubeName);
 // remove all existing segments
 CubeUpdate cubeBuilder = new CubeUpdate(cube);
@@ -187,7 +187,7 @@ public class BuildCubeWithStream {
 return job.getId();
 }
 
-private ExecutableState buildSegment(String cubeName, long startOffset, 
long endOffset) throws Exception {
+protected ExecutableState buildSegment(String cubeName, long startOffset, 
long endOffset) throws Exception {
 CubeSegment segment = 
cubeManager.appendSegment(cubeManager.getCube(cubeName), 0, 0, startOffset, 

[03/31] kylin git commit: KYLIN-1962 reorg BuildCubeWithStream test case

[liyang]

KYLIN-1962 reorg BuildCubeWithStream test case

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/859230d7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/859230d7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/859230d7

Branch: refs/heads/master-cdh5.7
Commit: 859230d787f9c218f56e56308897b68fb23d8dc4
Parents: ab5563a
Author: shaofengshi 
Authored: Mon Sep 26 18:10:32 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   2 +-
 .../java/org/apache/kylin/cube/CubeManager.java |  20 +++
 .../java/org/apache/kylin/cube/CubeSegment.java |   7 +-
 .../test_streaming_table_cube_desc.json |  17 ++-
 .../test_streaming_table_model_desc.json|   3 +-
 .../kylin/provision/BuildCubeWithStream.java| 121 +++--
 .../kylin/provision/BuildCubeWithStream2.java   | 134 ---
 7 files changed, 116 insertions(+), 188 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/859230d7/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3e41055..838ef97 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -807,6 +807,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getMaxBuildingSegments() {
-return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"1"));
+return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"2"));
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/859230d7/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 463c8e9..962568c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -29,10 +29,12 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.google.common.collect.Maps;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
@@ -476,6 +478,24 @@ public class CubeManager implements IRealizationProvider {
 if (pair.getFirst() == false || pair.getSecond() == false)
 throw new IllegalArgumentException("The new refreshing segment " + 
newSegment + " does not match any existing segment in cube " + cube);
 
+if (startOffset > 0 || endOffset > 0) {
+CubeSegment toRefreshSeg = null;
+for (CubeSegment cubeSegment : cube.getSegments()) {
+if (cubeSegment.getSourceOffsetStart() == startOffset && 
cubeSegment.getSourceOffsetEnd() == endOffset) {
+toRefreshSeg = cubeSegment;
+break;
+}
+}
+
+if (toRefreshSeg == null) {
+throw new IllegalArgumentException("For streaming cube, only 
one segment can be refreshed at one time");
+}
+
+Map partitionInfo = Maps.newHashMap();
+partitionInfo.putAll(toRefreshSeg.getAdditionalInfo());
+newSegment.setAdditionalInfo(partitionInfo);
+}
+
 CubeUpdate cubeBuilder = new CubeUpdate(cube);
 cubeBuilder.setToAddSegs(newSegment);
 updateCube(cubeBuilder);

http://git-wip-us.apache.org/repos/asf/kylin/blob/859230d7/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index afb0d28..d5de47f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -20,7 +20,6 @@ package org.apache.kylin.cube;
 
 import java.text.SimpleDateFormat;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -106,7 +105,7 @@ 

[20/31] kylin git commit: KYLIN-2054 TimedJsonStreamParser should support other time format

[liyang]

KYLIN-2054 TimedJsonStreamParser should support other time format


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e5007261
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e5007261
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e5007261

Branch: refs/heads/master-cdh5.7
Commit: e500726184b318da2d1f503cd1b159cfe7242347
Parents: 01d5670
Author: shaofengshi 
Authored: Tue Sep 27 22:21:15 2016 +0800
Committer: shaofengshi 
Committed: Wed Sep 28 17:00:46 2016 +0800

--
 build/bin/kylin.sh  |  3 +-
 .../org/apache/kylin/common/util/BasicTest.java | 13 +++
 source-kafka/pom.xml|  6 +-
 .../kylin/source/kafka/AbstractTimeParser.java  | 34 
 .../kylin/source/kafka/DateTimeParser.java  | 84 
 .../kylin/source/kafka/DefaultTimeParser.java   | 49 
 .../apache/kylin/source/kafka/KafkaMRInput.java |  2 +-
 .../kylin/source/kafka/SeekOffsetStep.java  |  2 +-
 .../source/kafka/TimedJsonStreamParser.java | 46 ---
 9 files changed, 222 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e5007261/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 7a9d2a1..e767492 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -31,7 +31,7 @@ function retrieveDependency() {
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh
 source ${dir}/find-hbase-dependency.sh
-source ${dir}/find-kafka-dependency.sh
+#source ${dir}/find-kafka-dependency.sh
 
 #retrive $KYLIN_EXTRA_START_OPTS
 if [ -f "${dir}/setenv.sh" ]
@@ -40,6 +40,7 @@ function retrieveDependency() {
 
 export 
HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
 export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
+#export 
HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}:${kafka_dependency}
 }
 
 # start command

http://git-wip-us.apache.org/repos/asf/kylin/blob/e5007261/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index ee15832..5eaa011 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -24,6 +24,7 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
@@ -33,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.LoggerFactory;
@@ -206,6 +208,17 @@ public class BasicTest {
 }
 }
 
+@Test
+@Ignore("for dev only")
+public void test3() throws Exception {
+FastDateFormat formatter = 
org.apache.kylin.common.util.DateFormat.getDateFormat("MMM dd,  hh:mm:ss 
aa");
+System.out.println(formatter.format(new Date()));
+
+String timeStr = "Jul 20, 2016 9:59:17 AM";
+
+System.out.println(formatter.parse(timeStr).getTime());
+}
+
 private static String time(long t) {
 DateFormat dateFormat = new SimpleDateFormat("/MM/dd HH:mm:ss");
 Calendar cal = Calendar.getInstance();

http://git-wip-us.apache.org/repos/asf/kylin/blob/e5007261/source-kafka/pom.xml
--
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index 212f4c6..f91ab8f 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -48,7 +48,11 @@
 kafka_2.10
 provided
 
-
+
+org.apache.commons
+commons-lang3
+${commons-lang3.version}
+
 
 
 org.apache.hadoop

http://git-wip-us.apache.org/repos/asf/kylin/blob/e5007261/source-kafka/src/main/java/org/apache/kylin/source/kafka/AbstractTimeParser.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/AbstractTimeParser.java
 

[02/31] kylin git commit: rename license header file

[liyang]

rename license header file


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a00d1e37
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a00d1e37
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a00d1e37

Branch: refs/heads/master-cdh5.7
Commit: a00d1e37483a5f15bf8f469fb1ed23bba1aa0011
Parents: afd3e61
Author: shaofengshi 
Authored: Tue Sep 27 09:55:03 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 09:55:03 2016 +0800

--
 dev-support/checkstyle-apache-header.txt | 17 +
 dev-support/checkstyle-java-header.txt   | 17 -
 dev-support/checkstyle.xml   |  2 +-
 3 files changed, 18 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a00d1e37/dev-support/checkstyle-apache-header.txt
--
diff --git a/dev-support/checkstyle-apache-header.txt 
b/dev-support/checkstyle-apache-header.txt
new file mode 100644
index 000..3740126
--- /dev/null
+++ b/dev-support/checkstyle-apache-header.txt
@@ -0,0 +1,17 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/a00d1e37/dev-support/checkstyle-java-header.txt
--
diff --git a/dev-support/checkstyle-java-header.txt 
b/dev-support/checkstyle-java-header.txt
deleted file mode 100644
index 3740126..000
--- a/dev-support/checkstyle-java-header.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/a00d1e37/dev-support/checkstyle.xml
--
diff --git a/dev-support/checkstyle.xml b/dev-support/checkstyle.xml
index ec36760..45325bf 100644
--- a/dev-support/checkstyle.xml
+++ b/dev-support/checkstyle.xml
@@ -21,7 +21,7 @@
 "http://www.puppycrawl.com/dtds/configuration_1_3.dtd;>
 
 
-
+
 
 
 

[01/31] kylin git commit: minor, add dev-support/sync_hbase_cdh_branches.cmd [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master-cdh5.7 beded5e2c -> 59beaa0ed (forced update)


minor, add dev-support/sync_hbase_cdh_branches.cmd


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/afd3e61e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/afd3e61e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/afd3e61e

Branch: refs/heads/master-cdh5.7
Commit: afd3e61eedea2ce175e3fb2844b6118aabe25a6f
Parents: 05afca7
Author: Yang Li 
Authored: Tue Sep 27 08:38:42 2016 +0800
Committer: Yang Li 
Committed: Tue Sep 27 08:38:42 2016 +0800

--
 dev-support/sync_hbase_cdh_branches.cmd | 19 +++
 1 file changed, 19 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/afd3e61e/dev-support/sync_hbase_cdh_branches.cmd
--
diff --git a/dev-support/sync_hbase_cdh_branches.cmd 
b/dev-support/sync_hbase_cdh_branches.cmd
new file mode 100644
index 000..0febfbb
--- /dev/null
+++ b/dev-support/sync_hbase_cdh_branches.cmd
@@ -0,0 +1,19 @@
+git checkout master
+git pull
+git reset apache/master --hard
+
+git checkout apache/1.5.x-HBase1.x
+git format-patch -1
+git checkout master
+git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
+git push apache master:1.5.x-HBase1.x -f
+rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
+
+git checkout apache/1.5.x-CDH5.7
+git format-patch -1
+git checkout master
+git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+git push apache master:1.5.x-CDH5.7 -f
+rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
+
+git reset apache/master --hard

[11/31] kylin git commit: Revert "Revert "rename the streaming_table.json""

[liyang]

Revert "Revert "rename the streaming_table.json""

This reverts commit bec25b4638835301a102141e56e12cd38d09139f.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8136380f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8136380f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8136380f

Branch: refs/heads/master-cdh5.7
Commit: 8136380f0aa6b0594b2882952e36077c6e474a3c
Parents: 25f8ffc
Author: shaofengshi 
Authored: Sat Sep 24 14:57:11 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../kafka/DEFAULT.STREAMING_TABLE.json  | 21 
 .../streaming/DEFAULT.STREAMING_TABLE.json  |  6 --
 2 files changed, 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8136380f/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
deleted file mode 100644
index 6a64cce..000
--- a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- 
-  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
-  "name": "DEFAULT.STREAMING_TABLE",
-  "topic": "test_streaming_table_topic_xyz",
-  "timeout": 6,
-  "bufferSize": 65536,
-  "parserName": "org.apache.kylin.source.kafka.TimedJsonStreamParser",
-  "last_modified": 0,
-  "clusters": [
-{
-  "brokers": [
-{
-  "id": 0,
-  "host": "sandbox",
-  "port": 6667
-}
-  ]
-}
-  ]
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/8136380f/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
deleted file mode 100644
index 85a477b..000
--- a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
-  "name": "DEFAULT.STREAMING_TABLE",
-  "type": "kafka",
-  "last_modified": 0
-}

[17/30] kylin git commit: KYLIN-2035 only auto-merge after a READY segment is built

[liyang]

KYLIN-2035 only auto-merge after a READY segment is built


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d366dc11
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d366dc11
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d366dc11

Branch: refs/heads/master-hbase1.x
Commit: d366dc1118aeb342a2ace251bc6dfac976f00e14
Parents: 859230d
Author: Li Yang 
Authored: Wed Sep 28 11:51:50 2016 +0800
Committer: Li Yang 
Committed: Wed Sep 28 11:51:50 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeInstance.java  | 12 
 .../org/apache/kylin/rest/service/CubeService.java | 13 -
 2 files changed, 20 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d366dc11/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index cad00e1..720690d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -293,6 +293,18 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return latest;
 }
 
+public CubeSegment getLatestBuiltSegment() {
+CubeSegment latest = null;
+for (int i = segments.size() - 1; i >= 0; i--) {
+CubeSegment seg = segments.get(i);
+if (seg.getLastBuildTime() > 0) {
+if (latest == null || seg.getLastBuildTime() > 
latest.getLastBuildTime())
+latest = seg;
+}
+}
+return latest;
+}
+
 public List getSegments() {
 return segments;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d366dc11/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index e446045..4fa07b6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -582,15 +582,18 @@ public class CubeService extends BasicService {
 }
 
 public void updateOnNewSegmentReady(String cubeName) {
-logger.debug("on updateOnNewSegmentReady: " + cubeName);
 final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 String serverMode = kylinConfig.getServerMode();
-logger.debug("server mode: " + serverMode);
 if (Constant.SERVER_MODE_JOB.equals(serverMode.toLowerCase()) || 
Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase())) {
-keepCubeRetention(cubeName);
-mergeCubeSegment(cubeName);
+CubeInstance cube = getCubeManager().getCube(cubeName);
+if (cube != null) {
+CubeSegment seg = cube.getLatestBuiltSegment();
+if (seg != null && seg.getStatus() == SegmentStatusEnum.READY) 
{
+keepCubeRetention(cubeName);
+mergeCubeSegment(cubeName);
+}
+}
 }
-
 }
 
 private void keepCubeRetention(String cubeName) {

[09/30] kylin git commit: Revert "Revert "KYLIN-1726 update to kafka 0.10""

[liyang]

Revert "Revert "KYLIN-1726 update to kafka 0.10""

This reverts commit 355e58ba4209ddf945663228688d550bf654c387.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae3d7e43
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae3d7e43
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae3d7e43

Branch: refs/heads/master-hbase1.x
Commit: ae3d7e4314bb5d5441d74224967c96a1f05f1b30
Parents: c738f0f
Author: shaofengshi 
Authored: Sat Sep 24 14:56:47 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 30 
 .../kylin/rest/controller/CubeController.java   |  8 ++---
 .../apache/kylin/rest/service/JobService.java   |  4 +--
 .../source/kafka/util/KafkaSampleProducer.java  | 38 
 4 files changed, 53 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ae3d7e43/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index cc2baa5..5a4b07c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -714,12 +714,28 @@ public class CubeManager implements IRealizationProvider {
 return null;
 }
 
-if (cube.getBuildingSegments().size() > 0) {
-logger.debug("Cube " + cube.getName() + " has bulding segment, 
will not trigger merge at this moment");
-return null;
+List buildingSegs = cube.getBuildingSegments();
+if (buildingSegs.size() > 0) {
+logger.debug("Cube " + cube.getName() + " has " + 
buildingSegs.size() + " building segments");
+}
+
+List readySegs = 
cube.getSegments(SegmentStatusEnum.READY);
+
+List mergingSegs = Lists.newArrayList();
+if (buildingSegs.size() > 0) {
+
+for (CubeSegment building : buildingSegs) {
+// exclude those under-merging segs
+for (CubeSegment ready : readySegs) {
+if (ready.getSourceOffsetStart() >= 
building.getSourceOffsetStart() && ready.getSourceOffsetEnd() <= 
building.getSourceOffsetEnd()) {
+mergingSegs.add(ready);
+}
+}
+}
 }
 
-List ready = cube.getSegments(SegmentStatusEnum.READY);
+// exclude those already under merging segments
+readySegs.removeAll(mergingSegs);
 
 long[] timeRanges = cube.getDescriptor().getAutoMergeTimeRanges();
 Arrays.sort(timeRanges);
@@ -727,9 +743,9 @@ public class CubeManager implements IRealizationProvider {
 for (int i = timeRanges.length - 1; i >= 0; i--) {
 long toMergeRange = timeRanges[i];
 
-for (int s = 0; s < ready.size(); s++) {
-CubeSegment seg = ready.get(s);
-Pair p = 
findMergeOffsetsByDateRange(ready.subList(s, ready.size()), //
+for (int s = 0; s < readySegs.size(); s++) {
+CubeSegment seg = readySegs.get(s);
+Pair p = 
findMergeOffsetsByDateRange(readySegs.subList(s, readySegs.size()), //
 seg.getDateRangeStart(), seg.getDateRangeStart() + 
toMergeRange, toMergeRange);
 if (p != null && p.getSecond().getDateRangeEnd() - 
p.getFirst().getDateRangeStart() >= toMergeRange)
 return Pair.newPair(p.getFirst().getSourceOffsetStart(), 
p.getSecond().getSourceOffsetEnd());

http://git-wip-us.apache.org/repos/asf/kylin/blob/ae3d7e43/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 42b117c..669f53e 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -272,7 +272,7 @@ public class CubeController extends BasicController {
 @RequestMapping(value = "/{cubeName}/rebuild", method = { 
RequestMethod.PUT })
 @ResponseBody
 public JobInstance rebuild(@PathVariable String cubeName, @RequestBody 
JobBuildRequest req) {
-return buildInternal(cubeName, req.getStartTime(), req.getEndTime(), 
0, 0, 

[03/30] kylin git commit: KYLIN-1962 reorg BuildCubeWithStream test case

[liyang]

KYLIN-1962 reorg BuildCubeWithStream test case

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/859230d7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/859230d7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/859230d7

Branch: refs/heads/master-hbase1.x
Commit: 859230d787f9c218f56e56308897b68fb23d8dc4
Parents: ab5563a
Author: shaofengshi 
Authored: Mon Sep 26 18:10:32 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   2 +-
 .../java/org/apache/kylin/cube/CubeManager.java |  20 +++
 .../java/org/apache/kylin/cube/CubeSegment.java |   7 +-
 .../test_streaming_table_cube_desc.json |  17 ++-
 .../test_streaming_table_model_desc.json|   3 +-
 .../kylin/provision/BuildCubeWithStream.java| 121 +++--
 .../kylin/provision/BuildCubeWithStream2.java   | 134 ---
 7 files changed, 116 insertions(+), 188 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/859230d7/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3e41055..838ef97 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -807,6 +807,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getMaxBuildingSegments() {
-return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"1"));
+return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"2"));
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/859230d7/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 463c8e9..962568c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -29,10 +29,12 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.google.common.collect.Maps;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinConfigExt;
@@ -476,6 +478,24 @@ public class CubeManager implements IRealizationProvider {
 if (pair.getFirst() == false || pair.getSecond() == false)
 throw new IllegalArgumentException("The new refreshing segment " + 
newSegment + " does not match any existing segment in cube " + cube);
 
+if (startOffset > 0 || endOffset > 0) {
+CubeSegment toRefreshSeg = null;
+for (CubeSegment cubeSegment : cube.getSegments()) {
+if (cubeSegment.getSourceOffsetStart() == startOffset && 
cubeSegment.getSourceOffsetEnd() == endOffset) {
+toRefreshSeg = cubeSegment;
+break;
+}
+}
+
+if (toRefreshSeg == null) {
+throw new IllegalArgumentException("For streaming cube, only 
one segment can be refreshed at one time");
+}
+
+Map partitionInfo = Maps.newHashMap();
+partitionInfo.putAll(toRefreshSeg.getAdditionalInfo());
+newSegment.setAdditionalInfo(partitionInfo);
+}
+
 CubeUpdate cubeBuilder = new CubeUpdate(cube);
 cubeBuilder.setToAddSegs(newSegment);
 updateCube(cubeBuilder);

http://git-wip-us.apache.org/repos/asf/kylin/blob/859230d7/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index afb0d28..d5de47f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -20,7 +20,6 @@ package org.apache.kylin.cube;
 
 import java.text.SimpleDateFormat;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -106,7 +105,7 

[18/30] kylin git commit: KYLIN-2050: Better close resource for AutoCloseable object

[liyang]

KYLIN-2050: Better close resource for AutoCloseable object


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/31b80905
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/31b80905
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/31b80905

Branch: refs/heads/master-hbase1.x
Commit: 31b80905ec3645ecf2c0e8bb5c02bdc451974648
Parents: d366dc1
Author: Yiming Liu 
Authored: Sun Sep 25 21:01:33 2016 +0800
Committer: Li Yang 
Committed: Wed Sep 28 13:37:26 2016 +0800

--
 .../org/apache/kylin/common/util/DBUtils.java   | 89 
 .../apache/kylin/common/util/ZipFileUtils.java  |  3 +-
 .../apache/kylin/dict/lookup/LookupTable.java   |  3 +-
 .../apache/kylin/metadata/MetadataManager.java  |  4 +-
 .../java/org/apache/kylin/query/QueryCli.java   | 13 +--
 .../kylin/query/enumerator/HiveEnumerator.java  | 19 +
 .../apache/kylin/rest/service/CubeService.java  |  5 +-
 .../apache/kylin/rest/service/QueryService.java | 23 +
 .../storage/hbase/steps/CubeHTableUtil.java |  7 +-
 .../storage/hbase/steps/DeprecatedGCStep.java   |  8 +-
 .../hbase/util/HBaseRegionSizeCalculator.java   |  3 +-
 .../storage/hbase/util/HbaseStreamingInput.java |  5 +-
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 13 +--
 13 files changed, 123 insertions(+), 72 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/31b80905/core-common/src/main/java/org/apache/kylin/common/util/DBUtils.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/DBUtils.java 
b/core-common/src/main/java/org/apache/kylin/common/util/DBUtils.java
new file mode 100644
index 000..9ecd3c3
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DBUtils.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DBUtils {
+
+private static final Logger logger = 
LoggerFactory.getLogger(DBUtils.class);
+
+/**
+ * Closes an ResultSet unconditionally.
+ * 
+ * Equivalent to {@link ResultSet#close()}, except any exceptions will be 
ignored.
+ * This is typically used in finally blocks.
+ * 
+ *
+ * @param output the ResultSet to close, may be null or already closed
+ */
+public static void closeQuietly(final ResultSet rs) {
+closeQuietly((AutoCloseable) rs);
+}
+
+/**
+ * Closes an Statement unconditionally.
+ * 
+ * Equivalent to {@link Statement#close()}, except any exceptions will be 
ignored.
+ * This is typically used in finally blocks.
+ * 
+ *
+ * @param output the ResultSet to close, may be null or already closed
+ */
+public static void closeQuietly(final Statement stmt) {
+closeQuietly((AutoCloseable) stmt);
+}
+
+/**
+ * Closes an Connection unconditionally.
+ * 
+ * Equivalent to {@link Connection#close()}, except any exceptions will be 
ignored.
+ * This is typically used in finally blocks.
+ * 
+ *
+ * @param output the ResultSet to close, may be null or already closed
+ */
+public static void closeQuietly(final Connection conn) {
+closeQuietly((AutoCloseable) conn);
+}
+
+/**
+ * Closes a AutoCloseable unconditionally.
+ * 
+ * Equivalent to {@link AutoCloseable#close()}, except any exceptions will 
be ignored. This is typically used in
+ * finally blocks.
+ * 
+ *
+ * @param closeable the objects to close, may be null or already closed
+ */
+public static void closeQuietly(final AutoCloseable closeable) {
+try {
+if (closeable != null) {
+closeable.close();
+}
+} catch (final Exception ioe) {
+logger.debug("", 

[19/30] kylin git commit: KYLIN-2042 respect KylinConfig.isQueryCacheEnabled(), thanks to Tajdar Siddiqui

[liyang]

KYLIN-2042 respect KylinConfig.isQueryCacheEnabled(), thanks to Tajdar Siddiqui


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01d5670e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01d5670e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01d5670e

Branch: refs/heads/master-hbase1.x
Commit: 01d5670e0ea7f1e3fe5cbb6e9decb293f8815d8f
Parents: 31b8090
Author: Li Yang 
Authored: Wed Sep 28 14:53:26 2016 +0800
Committer: Li Yang 
Committed: Wed Sep 28 14:53:26 2016 +0800

--
 .../kylin/rest/controller/QueryController.java  | 41 +++-
 1 file changed, 23 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/01d5670e/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 62cb0a7..9471937 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -173,7 +173,8 @@ public class QueryController extends BasicController {
 logger.info("Using project: " + project);
 logger.info("The original query:  " + sql);
 
-String serverMode = 
KylinConfig.getInstanceFromEnv().getServerMode();
+KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+String serverMode = kylinConfig.getServerMode();
 if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase()) 
|| Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase( {
 throw new InternalErrorException("Query is not allowed in " + 
serverMode + " mode.");
 }
@@ -185,17 +186,23 @@ public class QueryController extends BasicController {
 
 long startTime = System.currentTimeMillis();
 
-SQLResponse sqlResponse = searchQueryInCache(sqlRequest);
+SQLResponse sqlResponse = null;
+boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && 
!BackdoorToggles.getDisableCache();
+if (queryCacheEnabled) {
+sqlResponse = searchQueryInCache(sqlRequest);
+}
+
 try {
 if (null == sqlResponse) {
 sqlResponse = queryService.query(sqlRequest);
 
-long durationThreshold = 
KylinConfig.getInstanceFromEnv().getQueryDurationCacheThreshold();
-long scancountThreshold = 
KylinConfig.getInstanceFromEnv().getQueryScanCountCacheThreshold();
+long durationThreshold = 
kylinConfig.getQueryDurationCacheThreshold();
+long scancountThreshold = 
kylinConfig.getQueryScanCountCacheThreshold();
 sqlResponse.setDuration(System.currentTimeMillis() - 
startTime);
 logger.info("Stats of SQL response: isException: {}, 
duration: {}, total scan count {}", //
 String.valueOf(sqlResponse.getIsException()), 
String.valueOf(sqlResponse.getDuration()), 
String.valueOf(sqlResponse.getTotalScanCount()));
-if (!sqlResponse.getIsException() && 
(sqlResponse.getDuration() > durationThreshold || 
sqlResponse.getTotalScanCount() > scancountThreshold)) {
+if (queryCacheEnabled && !sqlResponse.getIsException() //
+&& (sqlResponse.getDuration() > durationThreshold 
|| sqlResponse.getTotalScanCount() > scancountThreshold)) {
 cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new 
Element(sqlRequest, sqlResponse));
 }
 } else {
@@ -211,7 +218,7 @@ public class QueryController extends BasicController {
 sqlResponse = new SQLResponse(null, null, 0, true, errMsg);
 
 // for exception queries, only cache ScanOutOfLimitException
-if (e instanceof ScanOutOfLimitException) {
+if (queryCacheEnabled && e instanceof ScanOutOfLimitException) 
{
 Cache exceptionCache = 
cacheManager.getCache(EXCEPTION_QUERY_CACHE);
 exceptionCache.put(new Element(sqlRequest, sqlResponse));
 }
@@ -236,18 +243,16 @@ public class QueryController extends BasicController {
 Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
 Cache successCache = cacheManager.getCache(SUCCESS_QUERY_CACHE);
 
-if (KylinConfig.getInstanceFromEnv().isQueryCacheEnabled() && 

[20/30] kylin git commit: KYLIN-2054 TimedJsonStreamParser should support other time format

[liyang]

KYLIN-2054 TimedJsonStreamParser should support other time format


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e5007261
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e5007261
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e5007261

Branch: refs/heads/master-hbase1.x
Commit: e500726184b318da2d1f503cd1b159cfe7242347
Parents: 01d5670
Author: shaofengshi 
Authored: Tue Sep 27 22:21:15 2016 +0800
Committer: shaofengshi 
Committed: Wed Sep 28 17:00:46 2016 +0800

--
 build/bin/kylin.sh  |  3 +-
 .../org/apache/kylin/common/util/BasicTest.java | 13 +++
 source-kafka/pom.xml|  6 +-
 .../kylin/source/kafka/AbstractTimeParser.java  | 34 
 .../kylin/source/kafka/DateTimeParser.java  | 84 
 .../kylin/source/kafka/DefaultTimeParser.java   | 49 
 .../apache/kylin/source/kafka/KafkaMRInput.java |  2 +-
 .../kylin/source/kafka/SeekOffsetStep.java  |  2 +-
 .../source/kafka/TimedJsonStreamParser.java | 46 ---
 9 files changed, 222 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e5007261/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 7a9d2a1..e767492 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -31,7 +31,7 @@ function retrieveDependency() {
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh
 source ${dir}/find-hbase-dependency.sh
-source ${dir}/find-kafka-dependency.sh
+#source ${dir}/find-kafka-dependency.sh
 
 #retrive $KYLIN_EXTRA_START_OPTS
 if [ -f "${dir}/setenv.sh" ]
@@ -40,6 +40,7 @@ function retrieveDependency() {
 
 export 
HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
 export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
+#export 
HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}:${kafka_dependency}
 }
 
 # start command

http://git-wip-us.apache.org/repos/asf/kylin/blob/e5007261/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java 
b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index ee15832..5eaa011 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -24,6 +24,7 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
@@ -33,6 +34,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import org.apache.commons.lang3.time.FastDateFormat;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.LoggerFactory;
@@ -206,6 +208,17 @@ public class BasicTest {
 }
 }
 
+@Test
+@Ignore("for dev only")
+public void test3() throws Exception {
+FastDateFormat formatter = 
org.apache.kylin.common.util.DateFormat.getDateFormat("MMM dd,  hh:mm:ss 
aa");
+System.out.println(formatter.format(new Date()));
+
+String timeStr = "Jul 20, 2016 9:59:17 AM";
+
+System.out.println(formatter.parse(timeStr).getTime());
+}
+
 private static String time(long t) {
 DateFormat dateFormat = new SimpleDateFormat("/MM/dd HH:mm:ss");
 Calendar cal = Calendar.getInstance();

http://git-wip-us.apache.org/repos/asf/kylin/blob/e5007261/source-kafka/pom.xml
--
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index 212f4c6..f91ab8f 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -48,7 +48,11 @@
 kafka_2.10
 provided
 
-
+
+org.apache.commons
+commons-lang3
+${commons-lang3.version}
+
 
 
 org.apache.hadoop

http://git-wip-us.apache.org/repos/asf/kylin/blob/e5007261/source-kafka/src/main/java/org/apache/kylin/source/kafka/AbstractTimeParser.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/AbstractTimeParser.java
 

[29/30] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/cad8d91c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/cad8d91c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[10/30] kylin git commit: Revert "Revert "refactor BuildCubeWithStream""

[liyang]

Revert "Revert "refactor BuildCubeWithStream""

This reverts commit 8e9c4550bb562b497442b17eec6485ae96e848d8.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be18158d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be18158d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be18158d

Branch: refs/heads/master-hbase1.x
Commit: be18158dcc5ce739c272b9345d3b2296c3936ee3
Parents: 8cbffb4
Author: shaofengshi 
Authored: Sat Sep 24 14:58:43 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |   7 +-
 .../kylin/provision/BuildCubeWithStream.java|  10 +-
 .../kylin/provision/BuildCubeWithStream2.java   | 145 +--
 3 files changed, 12 insertions(+), 150 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/be18158d/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 9b282e3..9e9df05 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -187,6 +187,7 @@ public class DeployUtil {
 File tmpFile = File.createTempFile(factTableName, "csv");
 FileOutputStream out = new FileOutputStream(tmpFile);
 
+InputStream tempIn = null;
 try {
 if (store.exists(factTablePath)) {
 InputStream oldContent = 
store.getResource(factTablePath).inputStream;
@@ -194,13 +195,15 @@ public class DeployUtil {
 }
 IOUtils.copy(in, out);
 IOUtils.closeQuietly(in);
+IOUtils.closeQuietly(out);
 
 store.deleteResource(factTablePath);
-in = new FileInputStream(tmpFile);
-store.putResource(factTablePath, in, System.currentTimeMillis());
+tempIn = new FileInputStream(tmpFile);
+store.putResource(factTablePath, tempIn, 
System.currentTimeMillis());
 } finally {
 IOUtils.closeQuietly(out);
 IOUtils.closeQuietly(in);
+IOUtils.closeQuietly(tempIn);
 }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/be18158d/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 6e5313f..bfe1d0a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -62,10 +62,10 @@ public class BuildCubeWithStream {
 
 private static final Logger logger = 
LoggerFactory.getLogger(org.apache.kylin.provision.BuildCubeWithStream.class);
 
-private CubeManager cubeManager;
+protected CubeManager cubeManager;
 private DefaultScheduler scheduler;
 protected ExecutableManager jobService;
-private static final String cubeName = "test_streaming_table_cube";
+static final String cubeName = "test_streaming_table_cube";
 
 private KafkaConfig kafkaConfig;
 private MockKafka kafkaServer;
@@ -114,13 +114,13 @@ public class BuildCubeWithStream {
 Assert.assertEquals(topicName, topicMetadata.topic());
 }
 
-private void generateStreamData(long startTime, long endTime, int 
numberOfRecords) throws IOException {
+protected void generateStreamData(long startTime, long endTime, int 
numberOfRecords) throws IOException {
 Kafka10DataLoader dataLoader = new Kafka10DataLoader(kafkaConfig);
 DeployUtil.prepareTestDataForStreamingCube(startTime, endTime, 
numberOfRecords, cubeName, dataLoader);
 logger.info("Test data inserted into Kafka");
 }
 
-private void clearSegment(String cubeName) throws Exception {
+protected void clearSegment(String cubeName) throws Exception {
 CubeInstance cube = cubeManager.getCube(cubeName);
 // remove all existing segments
 CubeUpdate cubeBuilder = new CubeUpdate(cube);
@@ -187,7 +187,7 @@ public class BuildCubeWithStream {
 return job.getId();
 }
 
-private ExecutableState buildSegment(String cubeName, long startOffset, 
long endOffset) throws Exception {
+protected ExecutableState buildSegment(String cubeName, long startOffset, 
long endOffset) throws Exception {
 CubeSegment segment = 
cubeManager.appendSegment(cubeManager.getCube(cubeName), 0, 0, startOffset, 

[05/30] kylin git commit: Revert "Revert "KYLIN-1818 change kafka dependency to provided""

[liyang]

Revert "Revert "KYLIN-1818 change kafka dependency to provided""

This reverts commit dee8f2d073d225a3b8189c66170fd310c1a8d221.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c738f0f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c738f0f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c738f0f2

Branch: refs/heads/master-hbase1.x
Commit: c738f0f265737dd5849f9342fd694e3cef3344b8
Parents: f89e35f
Author: shaofengshi 
Authored: Sat Sep 24 14:56:32 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 build/bin/find-kafka-dependency.sh  | 12 ++--
 build/bin/kylin.sh  |  2 ++
 .../kylin/engine/mr/common/AbstractHadoopJob.java   |  4 ++--
 3 files changed, 10 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c738f0f2/build/bin/find-kafka-dependency.sh
--
diff --git a/build/bin/find-kafka-dependency.sh 
b/build/bin/find-kafka-dependency.sh
index c6b9c24..7349360 100644
--- a/build/bin/find-kafka-dependency.sh
+++ b/build/bin/find-kafka-dependency.sh
@@ -32,20 +32,20 @@ then
 fi
 
 # works for kafka 9+
-kafka_client=`find -L "$(dirname $kafka_home)" -name 
'kafka-clients-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name 
'*sources*' ''-printf '%p:' | sed 's/:$//'`
-if [ -z "$kafka_client" ]
+kafka_dependency=`find -L $kafka_home -name 
'kafka-clients-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name 
'*sources*' ''-printf '%p:' | sed 's/:$//'`
+if [ -z "$kafka_dependency" ]
 then
 # works for kafka 8
-kafka_broker=`find -L "$(dirname $kafka_home)" -name 
'kafka_[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name 
'*sources*' ''-printf '%p:' | sed 's/:$//'`
-if [ -z "$kafka_broker" ]
+kafka_dependency=`find -L $kafka_home -name 'kafka_[a-z0-9A-Z\.-]*.jar' ! 
-name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 
's/:$//'`
+if [ -z "$kafka_dependency" ]
 then
 echo "kafka client lib not found"
 exit 1
 else
-echo "kafka dependency: $kafka_broker"
+echo "kafka dependency: $kafka_dependency"
 export kafka_dependency
 fi
 else
-echo "kafka dependency: $kafka_client"
+echo "kafka dependency: $kafka_dependency"
 export kafka_dependency
 fi

http://git-wip-us.apache.org/repos/asf/kylin/blob/c738f0f2/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 201b5b6..7a9d2a1 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -31,6 +31,7 @@ function retrieveDependency() {
 #retrive $hive_dependency and $hbase_dependency
 source ${dir}/find-hive-dependency.sh
 source ${dir}/find-hbase-dependency.sh
+source ${dir}/find-kafka-dependency.sh
 
 #retrive $KYLIN_EXTRA_START_OPTS
 if [ -f "${dir}/setenv.sh" ]
@@ -106,6 +107,7 @@ then
 -Djava.io.tmpdir=${tomcat_root}/temp  \
 -Dkylin.hive.dependency=${hive_dependency} \
 -Dkylin.hbase.dependency=${hbase_dependency} \
+-Dkylin.kafka.dependency=${kafka_dependency} \
 -Dkylin.rest.address=${kylin_rest_address} \
 -Dspring.profiles.active=${spring_profile} \
 org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar  
org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.out 
2>&1 & echo $! > ${KYLIN_HOME}/pid &

http://git-wip-us.apache.org/repos/asf/kylin/blob/c738f0f2/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index af2ed9f..a138eec 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -226,11 +226,11 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 }
 }
 
-// for hive dependencies
+// for kafka dependencies
 if (kylinKafkaDependency != null) {
 kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
 
-logger.info("Kafka Dependencies Before Filtered: " + 
kylinHiveDependency);
+logger.info("Kafka Dependencies Before Filtered: " + 
kylinKafkaDependency);
 
 if (kylinDependency.length() > 0)
 kylinDependency.append(",");

[14/30] kylin git commit: Revert "Revert "KYLIN-1726 allow job discard itself""

[liyang]

Revert "Revert "KYLIN-1726 allow job discard itself""

This reverts commit 3ae2549ba89e3a2c8ed94a2089678227cf78312d.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0de0239
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0de0239
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0de0239

Branch: refs/heads/master-hbase1.x
Commit: f0de02392f4808e08432417cb00f2ab5fa829055
Parents: de2f4e2
Author: shaofengshi 
Authored: Sat Sep 24 14:57:36 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java | 14 ++-
 .../kylin/job/execution/AbstractExecutable.java |  2 +
 .../job/execution/DefaultChainedExecutable.java |  2 +
 .../kylin/job/execution/ExecuteResult.java  |  4 ++
 .../kylin/job/DiscardedTestExecutable.java  | 41 
 .../impl/threadpool/DefaultSchedulerTest.java   | 16 
 6 files changed, 68 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f0de0239/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 5a4b07c..3a327f9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -357,34 +357,26 @@ public class CubeManager implements IRealizationProvider {
 Iterator iterator = newSegs.iterator();
 while (iterator.hasNext()) {
 CubeSegment currentSeg = iterator.next();
-boolean found = false;
 for (CubeSegment toRemoveSeg : update.getToRemoveSegs()) {
 if (currentSeg.getUuid().equals(toRemoveSeg.getUuid())) {
+logger.info("Remove segment " + currentSeg.toString());
+
toRemoveResources.add(currentSeg.getStatisticsResourcePath());
 iterator.remove();
-
toRemoveResources.add(toRemoveSeg.getStatisticsResourcePath());
-found = true;
+break;
 }
 }
-if (found == false) {
-logger.error("Segment '" + currentSeg.getName() + "' 
doesn't exist for remove.");
-}
 }
 
 }
 
 if (update.getToUpdateSegs() != null) {
 for (CubeSegment segment : update.getToUpdateSegs()) {
-boolean found = false;
 for (int i = 0; i < newSegs.size(); i++) {
 if (newSegs.get(i).getUuid().equals(segment.getUuid())) {
 newSegs.set(i, segment);
-found = true;
 break;
 }
 }
-if (found == false) {
-logger.error("Segment '" + segment.getName() + "' doesn't 
exist for update.");
-}
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f0de0239/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 90e4d3c..b4ca469 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -74,6 +74,8 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 if (!isDiscarded()) {
 if (result.succeed()) {
 executableManager.updateJobOutput(getId(), 
ExecutableState.SUCCEED, null, result.output());
+} else if (result.discarded()) {
+executableManager.updateJobOutput(getId(), 
ExecutableState.DISCARDED, null, result.output());
 } else {
 executableManager.updateJobOutput(getId(), 
ExecutableState.ERROR, null, result.output());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f0de0239/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 39a5f4f..5a57b05 100644

[15/30] kylin git commit: Revert "Revert "stop scheduler on test finish""

[liyang]

Revert "Revert "stop scheduler on test finish""

This reverts commit e604f6527d60b767f8a46a576765cfd205b1efc5.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8cbffb40
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8cbffb40
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8cbffb40

Branch: refs/heads/master-hbase1.x
Commit: 8cbffb40477611aa9c06bc5c22787aef83bc28d3
Parents: b1a0f4d
Author: shaofengshi 
Authored: Sat Sep 24 14:58:26 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../apache/kylin/job/impl/threadpool/BaseSchedulerTest.java| 1 +
 .../java/org/apache/kylin/provision/BuildCubeWithEngine.java   | 6 ++
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java   | 1 +
 .../java/org/apache/kylin/provision/BuildCubeWithStream2.java  | 1 +
 4 files changed, 9 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbffb40/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index 4e092a1..97c9f8d 100644
--- 
a/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ 
b/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -55,6 +55,7 @@ public abstract class BaseSchedulerTest extends 
LocalFileMetadataTestCase {
 
 @After
 public void after() throws Exception {
+DefaultScheduler.destroyInstance();
 cleanupTestMetadata();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbffb40/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 3d60a3c..31cf0eb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -79,6 +79,7 @@ public class BuildCubeWithEngine {
 BuildCubeWithEngine buildCubeWithEngine = new 
BuildCubeWithEngine();
 buildCubeWithEngine.before();
 buildCubeWithEngine.build();
+buildCubeWithEngine.after();
 logger.info("Build is done");
 afterClass();
 logger.info("Going to exit");
@@ -148,6 +149,11 @@ public class BuildCubeWithEngine {
 
 }
 
+
+public void after(){
+DefaultScheduler.destroyInstance();
+}
+
 public static void afterClass() {
 HBaseMetadataTestCase.staticCleanupTestMetadata();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbffb40/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index b7c609e..6e5313f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -217,6 +217,7 @@ public class BuildCubeWithStream {
 
 public void after() {
 kafkaServer.stop();
+DefaultScheduler.destroyInstance();
 }
 
 protected void waitForJob(String jobId) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8cbffb40/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
index d48a473..2812446 100644
--- 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
+++ 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream2.java
@@ -235,6 +235,7 @@ public class BuildCubeWithStream2 {
 
 public void after() {
 kafkaServer.stop();
+DefaultScheduler.destroyInstance();
 }
 
 protected void waitForJob(String jobId) {

[24/30] kylin git commit: enlarge the job interval in BuildCubeWithStream

[liyang]

enlarge the job interval in BuildCubeWithStream


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dfb41765
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dfb41765
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dfb41765

Branch: refs/heads/master-hbase1.x
Commit: dfb4176531046e212649dce89aebb8d091fc5689
Parents: 1e04981
Author: shaofengshi 
Authored: Fri Sep 30 11:02:09 2016 +0800
Committer: shaofengshi 
Committed: Fri Sep 30 11:02:09 2016 +0800

--
 .../java/org/apache/kylin/provision/BuildCubeWithStream.java | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dfb41765/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 23d7ca8..b48f75b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -22,9 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Random;
 import java.util.TimeZone;
 import java.util.UUID;
@@ -179,7 +177,7 @@ public class BuildCubeWithStream {
 
 List futures = Lists.newArrayList();
 for (int i = 0; i < 5; i++) {
-Thread.sleep(2 * 60 * 1000); // wait for new messages
+Thread.sleep(5 * 60 * 1000); // wait for new messages
 FutureTask futureTask = new FutureTask(new 
Callable() {
 @Override
 public ExecutableState call() {

[12/30] kylin git commit: Revert "Revert "KYLIN-1762 discard job when no stream message""

[liyang]

Revert "Revert "KYLIN-1762 discard job when no stream message""

This reverts commit da5ba276b972f8b3c0d220252e74ac2ff73298fc.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/25f8ffc0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/25f8ffc0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/25f8ffc0

Branch: refs/heads/master-hbase1.x
Commit: 25f8ffc0ee16b1702723d5530b22084a608fed9a
Parents: ae3d7e4
Author: shaofengshi 
Authored: Sat Sep 24 14:57:01 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../job/execution/DefaultChainedExecutable.java |  6 +++
 .../kylin/source/kafka/SeekOffsetStep.java  | 45 +++-
 2 files changed, 41 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/25f8ffc0/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 753b389..39a5f4f 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -88,6 +88,7 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 boolean allSucceed = true;
 boolean hasError = false;
 boolean hasRunning = false;
+boolean hasDiscarded = false;
 for (Executable task : jobs) {
 final ExecutableState status = task.getStatus();
 if (status == ExecutableState.ERROR) {
@@ -99,6 +100,9 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 if (status == ExecutableState.RUNNING) {
 hasRunning = true;
 }
+if (status == ExecutableState.DISCARDED) {
+hasDiscarded = true;
+}
 }
 if (allSucceed) {
 setEndTime(System.currentTimeMillis());
@@ -110,6 +114,8 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 notifyUserStatusChange(executableContext, 
ExecutableState.ERROR);
 } else if (hasRunning) {
 jobService.updateJobOutput(getId(), ExecutableState.RUNNING, 
null, null);
+} else if (hasDiscarded) {
+jobService.updateJobOutput(getId(), ExecutableState.DISCARDED, 
null, null);
 } else {
 jobService.updateJobOutput(getId(), ExecutableState.READY, 
null, null);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/25f8ffc0/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
index 5dca93f..479f1b8 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/SeekOffsetStep.java
@@ -17,6 +17,10 @@
 */
 package org.apache.kylin.source.kafka;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Maps;
+import org.apache.commons.math3.util.MathUtils;
 import org.apache.kylin.source.kafka.util.KafkaClient;
 import org.apache.kylin.source.kafka.util.KafkaOffsetMapping;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -34,6 +38,7 @@ import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
@@ -101,19 +106,39 @@ public class SeekOffsetStep extends AbstractExecutable {
 }
 }
 
-KafkaOffsetMapping.saveOffsetStart(segment, startOffsets);
-KafkaOffsetMapping.saveOffsetEnd(segment, endOffsets);
+long totalStartOffset = 0, totalEndOffset = 0;
+for (Long v : startOffsets.values()) {
+totalStartOffset += v;
+}
+for (Long v : endOffsets.values()) {
+totalEndOffset += v;
+}
 
-segment.setName(CubeSegment.makeSegmentName(0, 0, 
segment.getSourceOffsetStart(), segment.getSourceOffsetEnd()));
-CubeUpdate cubeBuilder = new CubeUpdate(cube);
-

[21/30] kylin git commit: minor add a blank line in create_sample_table.sql for beeline

[liyang]

minor add a blank line in create_sample_table.sql for beeline


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fd4ad58
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fd4ad58
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fd4ad58

Branch: refs/heads/master-hbase1.x
Commit: 0fd4ad587e3710324eaf58266b64f202eeabfa7e
Parents: e500726
Author: shaofengshi 
Authored: Wed Sep 28 17:51:36 2016 +0800
Committer: shaofengshi 
Committed: Wed Sep 28 17:51:36 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd4ad58/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index d83566c..5ed7b9b 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -187,4 +187,4 @@ STORED AS TEXTFILE;
 
 LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' 
OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES;
 LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' 
OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT;
-LOAD DATA INPATH 
'/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE 
INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;
\ No newline at end of file
+LOAD DATA INPATH 
'/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE 
INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;

[23/30] kylin git commit: minor, retry when http failed in job diagnosis

[liyang]

minor, retry when http failed in job diagnosis


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e049817
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e049817
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e049817

Branch: refs/heads/master-hbase1.x
Commit: 1e049817856ede06c7c8736ad1d608765f301a21
Parents: 51b5b31
Author: lidongsjtu 
Authored: Thu Sep 29 09:48:58 2016 +0800
Committer: lidongsjtu 
Committed: Thu Sep 29 15:34:25 2016 +0800

--
 .../java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java | 4 ++--
 .../java/org/apache/kylin/tool/JobTaskCounterExtractor.java | 9 +++--
 2 files changed, 9 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1e049817/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java 
b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
index 635a2c3..c8ff1f4 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
@@ -99,8 +99,8 @@ public class JobDiagnosisInfoCLI extends 
AbstractInfoExtractor {
 addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + jobId);
 addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + jobId);
 for (ExecutablePO task : executablePO.getTasks()) {
-addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + 
executablePO.getTasks());
-addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + 
executablePO.getTasks());
+addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + 
task.getUuid());
+addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + 
task.getUuid());
 if (includeYarnLogs) {
 yarnLogsResources.add(task.getUuid());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1e049817/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
--
diff --git 
a/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java 
b/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
index 7b9d8f3..6a317e9 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobTaskCounterExtractor.java
@@ -47,6 +47,8 @@ public class JobTaskCounterExtractor extends 
AbstractInfoExtractor {
 private String yarnUrl;
 private static final Logger logger = 
LoggerFactory.getLogger(JobTaskCounterExtractor.class);
 
+private final int HTTP_RETRY = 3;
+
 public JobTaskCounterExtractor(String mrJobId) {
 this.mrJobId = mrJobId;
 this.yarnUrl = getRestCheckUrl();
@@ -87,14 +89,17 @@ public class JobTaskCounterExtractor extends 
AbstractInfoExtractor {
 private String getHttpResponse(String url) {
 HttpClient client = new HttpClient();
 String response = null;
-while (response == null) {
+int retry_times = 0;
+while (response == null && retry_times < HTTP_RETRY) {
+retry_times++;
+
 HttpMethod get = new GetMethod(url);
 try {
 get.addRequestHeader("accept", "application/json");
 client.executeMethod(get);
 response = get.getResponseBodyAsString();
 } catch (Exception e) {
-logger.warn("Failed to fetch http response" + e);
+logger.warn("Failed to fetch http response. Retry={}", 
retry_times, e);
 } finally {
 get.releaseConnection();
 }

[13/30] kylin git commit: Revert "Revert "rename the streaming_table.json""

[liyang]

Revert "Revert "rename the streaming_table.json""

This reverts commit bec25b4638835301a102141e56e12cd38d09139f.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8136380f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8136380f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8136380f

Branch: refs/heads/master-hbase1.x
Commit: 8136380f0aa6b0594b2882952e36077c6e474a3c
Parents: 25f8ffc
Author: shaofengshi 
Authored: Sat Sep 24 14:57:11 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../kafka/DEFAULT.STREAMING_TABLE.json  | 21 
 .../streaming/DEFAULT.STREAMING_TABLE.json  |  6 --
 2 files changed, 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8136380f/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
deleted file mode 100644
index 6a64cce..000
--- a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- 
-  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
-  "name": "DEFAULT.STREAMING_TABLE",
-  "topic": "test_streaming_table_topic_xyz",
-  "timeout": 6,
-  "bufferSize": 65536,
-  "parserName": "org.apache.kylin.source.kafka.TimedJsonStreamParser",
-  "last_modified": 0,
-  "clusters": [
-{
-  "brokers": [
-{
-  "id": 0,
-  "host": "sandbox",
-  "port": 6667
-}
-  ]
-}
-  ]
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/8136380f/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
deleted file mode 100644
index 85a477b..000
--- a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
-  "name": "DEFAULT.STREAMING_TABLE",
-  "type": "kafka",
-  "last_modified": 0
-}

[22/30] kylin git commit: KYLIN-1962: Revert unsed split config feature

[liyang]

KYLIN-1962: Revert unsed split config feature

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/51b5b31b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/51b5b31b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/51b5b31b

Branch: refs/heads/master-hbase1.x
Commit: 51b5b31bbce3aac1d3a3c0de7746334961d5ffcd
Parents: 0fd4ad5
Author: Yiming Liu 
Authored: Tue Sep 27 07:52:52 2016 +0800
Committer: shaofengshi 
Committed: Wed Sep 28 17:51:49 2016 +0800

--
 .../org/apache/kylin/common/KylinConfig.java| 45 ---
 .../apache/kylin/common/KylinConfigTest.java| 12 
 .../test_case_data/localmeta/kylin.properties   | 33 ++-
 .../localmeta/kylin_account.properties  | 59 
 .../org/apache/kylin/tool/DiagnosisInfoCLI.java |  4 +-
 5 files changed, 33 insertions(+), 120 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/51b5b31b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index f134ad4..9543fc7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -47,7 +47,6 @@ public class KylinConfig extends KylinConfigBase {
 
 /** Kylin properties file name */
 public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties";
-public static final String KYLIN_ACCOUNT_CONF_PROPERTIES_FILE = 
"kylin_account.properties";
 public static final String KYLIN_CONF = "KYLIN_CONF";
 
 // static cached instances
@@ -205,23 +204,6 @@ public class KylinConfig extends KylinConfigBase {
 return getKylinPropertiesFile(path);
 }
 
-static File getKylinAccountPropertiesFile() {
-String kylinConfHome = System.getProperty(KYLIN_CONF);
-if (!StringUtils.isEmpty(kylinConfHome)) {
-logger.info("Use KYLIN_CONF=" + kylinConfHome);
-return getKylinAccountPropertiesFile(kylinConfHome);
-}
-
-logger.warn("KYLIN_CONF property was not set, will seek KYLIN_HOME env 
variable");
-
-String kylinHome = getKylinHome();
-if (StringUtils.isEmpty(kylinHome))
-throw new KylinConfigCannotInitException("Didn't find KYLIN_CONF 
or KYLIN_HOME, please set one of them");
-
-String path = kylinHome + File.separator + "conf";
-return getKylinAccountPropertiesFile(path);
-}
-
 public static Properties getKylinProperties() {
 File propFile = getKylinPropertiesFile();
 if (propFile == null || !propFile.exists()) {
@@ -242,25 +224,6 @@ public class KylinConfig extends KylinConfigBase {
 IOUtils.closeQuietly(ois);
 conf.putAll(propOverride);
 }
-
-File accountPropFile = getKylinAccountPropertiesFile();
-if (accountPropFile.exists()) {
-FileInputStream ois = new FileInputStream(accountPropFile);
-Properties propAccount = new Properties();
-propAccount.load(ois);
-IOUtils.closeQuietly(ois);
-conf.putAll(propAccount);
-}
-
-File accountPropOverrideFile = new 
File(accountPropFile.getParentFile(), accountPropFile.getName() + ".override");
-if (accountPropOverrideFile.exists()) {
-FileInputStream ois = new 
FileInputStream(accountPropOverrideFile);
-Properties propAccountOverride = new Properties();
-propAccountOverride.load(ois);
-IOUtils.closeQuietly(ois);
-conf.putAll(propAccountOverride);
-}
-
 } catch (IOException e) {
 throw new RuntimeException(e);
 }
@@ -282,14 +245,6 @@ public class KylinConfig extends KylinConfigBase {
 return new File(path, KYLIN_CONF_PROPERTIES_FILE);
 }
 
-private static File getKylinAccountPropertiesFile(String path) {
-if (path == null) {
-return null;
-}
-
-return new File(path, KYLIN_ACCOUNT_CONF_PROPERTIES_FILE);
-}
-
 public static void setSandboxEnvIfPossible() {
 File dir1 = new File("../examples/test_case_data/sandbox");
 File dir2 = new File("../../kylin/examples/test_case_data/sandbox");

http://git-wip-us.apache.org/repos/asf/kylin/blob/51b5b31b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java

[25/30] kylin git commit: fix it, run StorageCleanup after finish BuildCubeWithStream

[liyang]

fix it, run StorageCleanup after finish BuildCubeWithStream

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a6b1f2d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a6b1f2d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a6b1f2d

Branch: refs/heads/master-hbase1.x
Commit: 3a6b1f2d1cd1a42837353e52e3fa219c0d5a314b
Parents: dfb4176
Author: shaofengshi 
Authored: Fri Sep 30 14:04:12 2016 +0800
Committer: shaofengshi 
Committed: Fri Sep 30 14:04:12 2016 +0800

--
 .../org/apache/kylin/provision/BuildCubeWithStream.java  | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3a6b1f2d/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index b48f75b..58715f1 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit;
 import com.google.common.collect.Lists;
 import org.I0Itec.zkclient.ZkConnection;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.util.ToolRunner;
 import org.apache.kafka.common.requests.MetadataResponse;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.ClassUtil;
@@ -58,6 +59,7 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.source.kafka.KafkaConfigManager;
 import org.apache.kylin.source.kafka.config.BrokerConfig;
 import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
 import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
 import org.junit.Assert;
 import org.slf4j.Logger;
@@ -177,7 +179,7 @@ public class BuildCubeWithStream {
 
 List futures = Lists.newArrayList();
 for (int i = 0; i < 5; i++) {
-Thread.sleep(5 * 60 * 1000); // wait for new messages
+Thread.sleep(2 * 60 * 1000); // wait for new messages
 FutureTask futureTask = new FutureTask(new 
Callable() {
 @Override
 public ExecutableState call() {
@@ -276,6 +278,7 @@ public class BuildCubeWithStream {
 }
 
 public static void afterClass() throws Exception {
+cleanupOldStorage();
 HBaseMetadataTestCase.staticCleanupTestMetadata();
 }
 
@@ -299,6 +302,12 @@ public class BuildCubeWithStream {
 }
 }
 
+private static void cleanupOldStorage() throws Exception {
+String[] args = { "--delete", "true" };
+StorageCleanupJob cli = new StorageCleanupJob();
+cli.execute(args);
+}
+
 public static void main(String[] args) throws Exception {
 try {
 beforeClass();

[26/30] kylin git commit: minor, fix sample cube desc

[liyang]

minor, fix sample cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92fc0e53
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92fc0e53
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92fc0e53

Branch: refs/heads/master-hbase1.x
Commit: 92fc0e53845a30a320467e3657a90343d2002e8c
Parents: 3a6b1f2
Author: Yang Li 
Authored: Fri Sep 30 21:46:31 2016 +0800
Committer: Yang Li 
Committed: Fri Sep 30 21:48:35 2016 +0800

--
 .../org/apache/kylin/engine/mr/HadoopUtil.java  |  3 +-
 .../cube_desc/kylin_sales_cube_desc.json| 46 ++--
 2 files changed, 24 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/92fc0e53/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 0ed39d5..88692a0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -34,6 +34,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class HadoopUtil {
+@SuppressWarnings("unused")
 private static final Logger logger = 
LoggerFactory.getLogger(HadoopUtil.class);
 private static final ThreadLocal hadoopConfig = new 
ThreadLocal<>();
 
@@ -45,11 +46,9 @@ public class HadoopUtil {
 if (hadoopConfig.get() == null) {
 Configuration conf = healSickConfig(new Configuration());
 // do not cache this conf, or will affect following mr jobs
-logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 return conf;
 }
 Configuration conf = hadoopConfig.get();
-logger.info("The conf for current mapper will be " + 
System.identityHashCode(conf));
 return conf;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/92fc0e53/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index 70b1294..99f8f46 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -1,7 +1,10 @@
 {
   "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+  "last_modified" : 1451468470824,
   "name" : "kylin_sales_cube_desc",
+  "model_name" : "kylin_sales_model",
   "description" : null,
+  "null_string" : null,
   "dimensions" : [ {
 "name" : "CAL_DT",
 "table" : "DEFAULT.KYLIN_CAL_DT",
@@ -106,21 +109,21 @@
 },
 "dependent_measure_ref" : null
   }, {
-"name": "TOP_SELLER",
-"function": {
-  "expression": "TOP_N",
-  "parameter": {
-"type": "column",
-"value": "PRICE",
-"next_parameter": {
-  "type": "column",
-  "value": "SELLER_ID",
-  "next_parameter": null
+"name" : "TOP_SELLER",
+"function" : {
+  "expression" : "TOP_N",
+  "parameter" : {
+"type" : "column",
+"value" : "PRICE",
+"next_parameter" : {
+  "type" : "column",
+  "value" : "SELLER_ID",
+  "next_parameter" : null
 }
   },
-  "returntype": "topn(100)"
+  "returntype" : "topn(100)"
 },
-"dependent_measure_ref": null
+"dependent_measure_ref" : null
   } ],
   "rowkey" : {
 "rowkey_columns" : [ {
@@ -146,21 +149,18 @@
   "encoding" : "dict"
 } ]
   },
-  "last_modified" : 1451468470824,
-  "model_name" : "kylin_sales_model",
-  "null_string" : null,
   "hbase_mapping" : {
 "column_family" : [ {
-  "name" : "f1",
+  "name" : "F1",
   "columns" : [ {
-"qualifier" : "m",
-"measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
+"qualifier" : "M",
+"measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", 
"TOP_SELLER" ]
   } ]
 }, {
-  "name" : "f2",
+  "name" : "F2",
   "columns" : [ {
-"qualifier" : "m",
-"measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT", "TOP_SELLER" 
]
+"qualifier" : "M",
+"measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT" ]
   } ]
 } ]
   },
@@ -174,10 +174,10 @@
   } ],
   "notify_list" : null,
   "status_need_notify" : [ ],
+  "partition_date_start" : 132537600,
   "auto_merge_time_ranges" : null,
   "retention_range" : 0,
   "engine_type" : 

[11/30] kylin git commit: Revert "Revert "change to upper case""

[liyang]

Revert "Revert "change to upper case""

This reverts commit d1e979b4995597ae2ac3fbeb88ba1902d7296782.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/de2f4e2a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/de2f4e2a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/de2f4e2a

Branch: refs/heads/master-hbase1.x
Commit: de2f4e2a395af57a42aba43f7233bd1bbdd0f6d0
Parents: 8136380
Author: shaofengshi 
Authored: Sat Sep 24 14:57:22 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../kafka/DEFAULT.STREAMING_TABLE.json  | 21 
 .../streaming/DEFAULT.STREAMING_TABLE.json  |  6 ++
 2 files changed, 27 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/de2f4e2a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
new file mode 100644
index 000..6a64cce
--- /dev/null
+++ b/examples/test_case_data/localmeta/kafka/DEFAULT.STREAMING_TABLE.json
@@ -0,0 +1,21 @@
+{
+ 
+  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
+  "name": "DEFAULT.STREAMING_TABLE",
+  "topic": "test_streaming_table_topic_xyz",
+  "timeout": 6,
+  "bufferSize": 65536,
+  "parserName": "org.apache.kylin.source.kafka.TimedJsonStreamParser",
+  "last_modified": 0,
+  "clusters": [
+{
+  "brokers": [
+{
+  "id": 0,
+  "host": "sandbox",
+  "port": 6667
+}
+  ]
+}
+  ]
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/de2f4e2a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
--
diff --git 
a/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json 
b/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
new file mode 100644
index 000..85a477b
--- /dev/null
+++ b/examples/test_case_data/localmeta/streaming/DEFAULT.STREAMING_TABLE.json
@@ -0,0 +1,6 @@
+{
+  "uuid": "8b2b9dfe-777c-4d39-bf89-8472ec909193",
+  "name": "DEFAULT.STREAMING_TABLE",
+  "type": "kafka",
+  "last_modified": 0
+}

[07/30] kylin git commit: Revert "Revert "KYLIN-1726 Scalable streaming cubing""

[liyang]

Revert "Revert "KYLIN-1726 Scalable streaming cubing""

This reverts commit 506cd783132023a06f1669ad248b74bf9d96d0e1.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8431af45
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8431af45
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8431af45

Branch: refs/heads/master-hbase1.x
Commit: 8431af45528abb2d39a69b5e762712983573e5a6
Parents: a00d1e3
Author: shaofengshi 
Authored: Sat Sep 24 14:55:59 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../java/org/apache/kylin/job/DeployUtil.java   |  35 ++-
 .../kylin/job/streaming/Kafka10DataLoader.java  |  80 +++
 .../apache/kylin/common/KylinConfigBase.java|   1 +
 .../java/org/apache/kylin/cube/CubeSegment.java |   1 +
 .../java/org/apache/kylin/cube/ISegment.java|  39 
 .../cube/gridtable/SegmentGTStartAndEnd.java|   2 +-
 .../cube/model/CubeJoinedFlatTableDesc.java |   6 +
 .../cube/model/CubeJoinedFlatTableEnrich.java   |   6 +
 .../apache/kylin/gridtable/ScannerWorker.java   |   2 +-
 .../metadata/model/IJoinedFlatTableDesc.java|   2 +
 .../apache/kylin/metadata/model/ISegment.java   |  36 +++
 .../kylin/engine/mr/BatchMergeJobBuilder2.java  |   3 +
 .../org/apache/kylin/engine/mr/IMRInput.java|  10 +
 .../java/org/apache/kylin/engine/mr/MRUtil.java |   4 +
 .../test_streaming_table_model_desc.json|   6 +-
 .../kylin/provision/BuildCubeWithStream.java| 218 +-
 .../org/apache/kylin/provision/MockKafka.java   | 191 
 .../apache/kylin/provision/NetworkUtils.java|  52 +
 pom.xml |   2 +-
 .../apache/kylin/source/hive/HiveMRInput.java   |  11 +
 source-kafka/pom.xml|  13 +-
 .../apache/kylin/source/kafka/KafkaMRInput.java | 221 +++
 .../apache/kylin/source/kafka/KafkaSource.java  |  57 +
 .../kylin/source/kafka/KafkaStreamingInput.java |  17 +-
 .../kylin/source/kafka/MergeOffsetStep.java |  89 
 .../kylin/source/kafka/SeekOffsetStep.java  | 119 ++
 .../kylin/source/kafka/UpdateTimeRangeStep.java | 108 +
 .../source/kafka/config/KafkaClusterConfig.java |   3 +-
 .../source/kafka/hadoop/KafkaFlatTableJob.java  | 165 ++
 .../kafka/hadoop/KafkaFlatTableMapper.java  |  51 +
 .../source/kafka/hadoop/KafkaInputFormat.java   |  98 
 .../kafka/hadoop/KafkaInputRecordReader.java| 166 ++
 .../source/kafka/hadoop/KafkaInputSplit.java| 102 +
 .../kylin/source/kafka/util/KafkaClient.java| 115 ++
 .../source/kafka/util/KafkaOffsetMapping.java   |  97 
 .../kylin/source/kafka/util/KafkaRequester.java |   7 +-
 .../kylin/source/kafka/util/KafkaUtils.java |   3 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java |   2 +-
 .../storage/hbase/cube/v2/CubeHBaseRPC.java |   2 +-
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |   2 +-
 40 files changed, 2024 insertions(+), 120 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8431af45/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java 
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 8c64f91..9b282e3 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -143,14 +143,12 @@ public class DeployUtil {
 deployHiveTables();
 }
 
-public static void prepareTestDataForStreamingCube(long startTime, long 
endTime, String cubeName, StreamDataLoader streamDataLoader) throws IOException 
{
+public static void prepareTestDataForStreamingCube(long startTime, long 
endTime, int numberOfRecords, String cubeName, StreamDataLoader 
streamDataLoader) throws IOException {
 CubeInstance cubeInstance = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
-List data = StreamingTableDataGenerator.generate(1, 
startTime, endTime, cubeInstance.getFactTable());
-List data2 = StreamingTableDataGenerator.generate(10, endTime, 
endTime + 30, cubeInstance.getFactTable());
+List data = 
StreamingTableDataGenerator.generate(numberOfRecords, startTime, endTime, 
cubeInstance.getFactTable());
 TableDesc tableDesc = cubeInstance.getFactTableDesc();
 //load into kafka
 streamDataLoader.loadIntoKafka(data);
-streamDataLoader.loadIntoKafka(data2);
 logger.info("Write {} messages into {}", data.size(), 
streamDataLoader.toString());
 
 //csv 

[30/30] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cad8d91c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cad8d91c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cad8d91c

Branch: refs/heads/master-hbase1.x
Commit: cad8d91c3ad23eb0b53788ceed59a8379b79cea6
Parents: 937cc54
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 08:31:03 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 461 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cad8d91c/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[08/30] kylin git commit: Revert "Revert "KYLIN-1726 use segment uuid instead of name""

[liyang]

Revert "Revert "KYLIN-1726 use segment uuid instead of name""

This reverts commit 1f4880479cd3132786062723ba70312440de4805.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f89e35f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f89e35f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f89e35f6

Branch: refs/heads/master-hbase1.x
Commit: f89e35f6309c9bec43cc16e68fb35b7490aecc38
Parents: 8431af4
Author: shaofengshi 
Authored: Sat Sep 24 14:56:17 2016 +0800
Committer: shaofengshi 
Committed: Tue Sep 27 10:17:40 2016 +0800

--
 .../java/org/apache/kylin/cube/CubeManager.java |  5 +++-
 .../kylin/provision/BuildCubeWithStream.java| 26 +---
 .../apache/kylin/source/kafka/KafkaMRInput.java |  2 +-
 .../source/kafka/hadoop/KafkaFlatTableJob.java  | 11 +++--
 .../kafka/hadoop/KafkaInputRecordReader.java|  9 ---
 5 files changed, 36 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f89e35f6/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 2fadedb..cc2baa5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -479,8 +479,11 @@ public class CubeManager implements IRealizationProvider {
 updateCube(cubeBuilder);
 return newSegment;
 }
-
 public CubeSegment refreshSegment(CubeInstance cube, long startDate, long 
endDate, long startOffset, long endOffset) throws IOException {
+return refreshSegment(cube, startDate, endDate, startOffset, 
endOffset, true);
+}
+
+public CubeSegment refreshSegment(CubeInstance cube, long startDate, long 
endDate, long startOffset, long endOffset, boolean strictChecking) throws 
IOException {
 checkNoBuildingSegment(cube);
 
 CubeSegment newSegment = newSegment(cube, startDate, endDate, 
startOffset, endOffset);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f89e35f6/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 7f79acc..9e779ab 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -21,6 +21,8 @@ package org.apache.kylin.provision;
 import java.io.File;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
+import java.util.HashMap;
+import java.util.List;
 import java.util.TimeZone;
 import java.util.UUID;
 
@@ -145,18 +147,34 @@ public class BuildCubeWithStream {
 //merge
 mergeSegment(cubeName, 0, 15000);
 
+List segments = 
cubeManager.getCube(cubeName).getSegments();
+Assert.assertTrue(segments.size() == 1);
+
+CubeSegment toRefreshSeg = segments.get(0);
+HashMap partitionOffsetMap = 
toRefreshSeg.getAdditionalInfo();
+
+refreshSegment(cubeName, toRefreshSeg.getSourceOffsetStart(), 
toRefreshSeg.getSourceOffsetEnd(), partitionOffsetMap);
+segments = cubeManager.getCube(cubeName).getSegments();
+Assert.assertTrue(segments.size() == 1);
+
 }
 
 private String mergeSegment(String cubeName, long startOffset, long 
endOffset) throws Exception {
-CubeSegment segment = 
cubeManager.mergeSegments(cubeManager.getCube(cubeName), 0, 0, startOffset, 
endOffset, true);
+CubeSegment segment = 
cubeManager.mergeSegments(cubeManager.getCube(cubeName), 0, 0, startOffset, 
endOffset, false);
 DefaultChainedExecutable job = 
EngineFactory.createBatchMergeJob(segment, "TEST");
 jobService.addJob(job);
 waitForJob(job.getId());
 return job.getId();
 }
 
-private String refreshSegment(String cubeName, long startOffset, long 
endOffset) throws Exception {
-CubeSegment segment = 
cubeManager.refreshSegment(cubeManager.getCube(cubeName), 0, 0, startOffset, 
endOffset);
+private String refreshSegment(String cubeName, long startOffset, long 
endOffset, HashMap partitionOffsetMap) throws Exception {
+CubeSegment segment = 
cubeManager.refreshSegment(cubeManager.getCube(cubeName), 0, 0, startOffset, 
endOffset, false);
+segment.setAdditionalInfo(partitionOffsetMap);
+CubeInstance cubeInstance = 

kylin git commit: minor, enhance dev-support scripts

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master e22039d13 -> 937cc546d


minor, enhance dev-support scripts


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/937cc546
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/937cc546
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/937cc546

Branch: refs/heads/master
Commit: 937cc546d1dc34832217f1b806f3e91aa486caa4
Parents: e22039d
Author: Yang Li 
Authored: Tue Oct 4 15:45:45 2016 +0800
Committer: Yang Li 
Committed: Tue Oct 4 15:45:45 2016 +0800

--
 dev-support/sync_hbase_cdh_branches.cmd   | 44 +-
 dev-support/test_all_against_hdp_2_2_4_2_2.sh |  2 +-
 2 files changed, 35 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/937cc546/dev-support/sync_hbase_cdh_branches.cmd
--
diff --git a/dev-support/sync_hbase_cdh_branches.cmd 
b/dev-support/sync_hbase_cdh_branches.cmd
index 0febfbb..ed828ca 100644
--- a/dev-support/sync_hbase_cdh_branches.cmd
+++ b/dev-support/sync_hbase_cdh_branches.cmd
@@ -1,19 +1,43 @@
-git checkout master
-git pull
-git reset apache/master --hard
+#!/bin/bash
+
+# 
+
+base=master
+
+# 
+
+set -o pipefail  # trace ERR through pipes
+set -o errtrace  # trace ERR through 'time command' and other functions
+function error() {
+   SCRIPT="$0"   # script name
+   LASTLINE="$1" # line of error occurrence
+   LASTERR="$2"  # error code
+   echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code 
${LASTERR}"
+   exit 1
+}
+trap 'error ${LINENO} ${?}' ERR
+
+# 
 
-git checkout apache/1.5.x-HBase1.x
+git fetch apache
+git checkout apache/$base-hbase1.x
 git format-patch -1
-git checkout master
+git checkout apache/$base-cdh1.x
+git format-patch -1
+
+git checkout apache/$base
+git checkout -b tmp
+git reset apache/$base --hard
+
 git am -3 --ignore-whitespace 
0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
-git push apache master:1.5.x-HBase1.x -f
+#git push apache tmp:$base-hbase1.x -f
 rm 0001-KYLIN-1528-Create-a-branch-for-v1.5-with-HBase-1.x-A.patch
 
-git checkout apache/1.5.x-CDH5.7
-git format-patch -1
-git checkout master
 git am -3 --ignore-whitespace 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
-git push apache master:1.5.x-CDH5.7 -f
+#git push apache tmp:$base-cdh5.7 -f
 rm 0001-KYLIN-1672-support-kylin-on-cdh-5.7.patch
 
+# clean up
+git checkout master
 git reset apache/master --hard
+git checkout -b tmp

http://git-wip-us.apache.org/repos/asf/kylin/blob/937cc546/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
index f0d27e4..f7780dd 100644
--- a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -22,4 +22,4 @@ cd ${dir}
 cd ..
 
 mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=2.2.4.2-2 -fae 2>&1 | tee mvnverify.log
+mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee 
mvnverify.log

[kylin] Git Push Summary

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/1.5.x-CDH5.7 [deleted] beded5e2c

[kylin] Git Push Summary

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master-hbase1.x [created] 787b0aad7

[kylin] Git Push Summary

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/1.5.x-hbase1.x [deleted] 787b0aad7

[kylin] Git Push Summary

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/1.5.x-HBase1.x [deleted] 787b0aad7

[kylin] Git Push Summary

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/1.5.x-hbase1.x [created] 787b0aad7