kylin git commit: KYLIN-2099 fix sample table/cube desc

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 3924672be -> 5429006a5


KYLIN-2099 fix sample table/cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5429006a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5429006a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5429006a

Branch: refs/heads/master
Commit: 5429006a5976a52a08ef39976dfedfeb164629ea
Parents: 3924672
Author: Li Yang 
Authored: Mon Oct 17 16:42:44 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 16:45:31 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql   | 16 
 .../template/cube_desc/kylin_sales_cube_desc.json   |  2 +-
 .../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json |  4 ++--
 3 files changed, 11 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index fa5e25b..9fa17d7 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -116,10 +116,10 @@ CAL_DT date COMMENT 'Date, PK'
 ,YTM_YN_ID tinyint
 ,YTQ_YN_ID tinyint
 ,YTW_YN_ID tinyint
-,CRE_DATE string
-,CRE_USER string
-,UPD_DATE string
-,UPD_USER string
+,KYLIN_CAL_DT_CRE_DATE string
+,KYLIN_CAL_DT_CRE_USER string
+,KYLIN_CAL_DT_UPD_DATE string
+,KYLIN_CAL_DT_UPD_USER string
 )
 COMMENT 'Date Dimension Table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
@@ -137,10 +137,10 @@ LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
 ,REGN_CATEG string
 ,USER_DEFINED_FIELD1 string COMMENT 'User Defined Field1'
 ,USER_DEFINED_FIELD3 string COMMENT 'User Defined Field3'
-,CRE_DATE string
-,UPD_DATE string COMMENT 'Last Updated Date'
-,CRE_USER string
-,UPD_USER string COMMENT 'Last Updated User'
+,KYLIN_GROUPINGS_CRE_DATE string
+,KYLIN_GROUPINGS_UPD_DATE string COMMENT 'Last Updated Date'
+,KYLIN_GROUPINGS_CRE_USER string
+,KYLIN_GROUPINGS_UPD_USER string COMMENT 'Last Updated User'
 ,META_CATEG_ID decimal
 ,META_CATEG_NAME string COMMENT 'Level1 Category'
 ,CATEG_LVL2_ID decimal

http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index cc1cd76..3e9bff4 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -14,7 +14,7 @@
 "name" : "CATEGORY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "{FK}",
-"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER" ]
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
   }, {
 "name" : "CATEGORY_HIERARCHY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",

http://git-wip-us.apache.org/repos/asf/kylin/blob/5429006a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
--
diff --git 
a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json 
b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
index 8e97309..accba35 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
@@ -40,7 +40,7 @@
 "datatype" : "string"
   }, {
 "id" : "10",
-"name" : "UPD_DATE",
+"name" : "KYLIN_GROUPINGS_UPD_DATE",
 "datatype" : "string"
   }, {
 "id" : "11",
@@ -48,7 +48,7 @@
 "datatype" : "string"
   }, {
 "id" : "12",
-"name" : "UPD_USER",
+"name" : "KYLIN_GROUPINGS_UPD_USER",
 "datatype" : "string"
   }, {
 "id" : "13",

kylin git commit: KYLIN-2091 Add API to init the start-point (of each parition) for streaming cube

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master 5429006a5 -> c92f79ad3


KYLIN-2091 Add API to init the start-point (of each parition) for streaming cube

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c92f79ad
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c92f79ad
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c92f79ad

Branch: refs/heads/master
Commit: c92f79ad39e562ed32fff1a55eb979f0593ed6e3
Parents: 5429006
Author: shaofengshi 
Authored: Mon Oct 17 17:40:57 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 17:53:56 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  2 +-
 .../java/org/apache/kylin/cube/CubeSegment.java | 12 ++
 .../org/apache/kylin/cube/model/CubeDesc.java   | 13 ++
 .../kylin/measure/topn/TopNMeasureType.java |  2 +-
 server-base/pom.xml |  7 
 .../kylin/rest/controller/CubeController.java   | 10 ++---
 .../rest/controller/CubeDescController.java | 44 +++-
 .../kylin/source/kafka/job/SeekOffsetStep.java  |  8 +++-
 .../kylin/source/kafka/util/KafkaClient.java| 22 +-
 9 files changed, 108 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4942081..73ac788 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -804,7 +804,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getMaxBuildingSegments() {
-return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"2"));
+return Integer.parseInt(getOptional("kylin.cube.building.segment.max", 
"10"));
 }
 
 public void setMaxBuildingSegments(int maxBuildingSegments) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index fdf1fb0..eb5b389 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -564,6 +564,12 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetEnd(Map 
sourcePartitionOffsetEnd) {
 this.sourcePartitionOffsetEnd = sourcePartitionOffsetEnd;
+long total = 0;
+for (Long x : sourcePartitionOffsetEnd.values()) {
+total += x;
+}
+
+this.sourceOffsetEnd = total;
 }
 
 public Map getSourcePartitionOffsetStart() {
@@ -572,5 +578,11 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetStart(Map 
sourcePartitionOffsetStart) {
 this.sourcePartitionOffsetStart = sourcePartitionOffsetStart;
+long total = 0;
+for (Long x : sourcePartitionOffsetStart.values()) {
+total += x;
+}
+
+this.sourceOffsetStart = total;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c92f79ad/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..bf3724a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -170,6 +170,10 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 
 private Map extendedColumnToHosts = 
Maps.newHashMap();
 
+@JsonProperty("partition_offset_start")
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+private Map partitionOffsetStart = Maps.newHashMap();
+
 public boolean isEnableSharding() {
 //in the future may extend to other storage that is shard-able
 return storageType != IStorageAware.ID_HBASE && storageType != 
IStorageAware.ID_HYBRID;
@@ -1011,6 +1015,14 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
   

kylin git commit: minor, mapreduce speculative off by default

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21 e27f545fb -> 8006bb135


minor, mapreduce speculative off by default


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8006bb13
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8006bb13
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8006bb13

Branch: refs/heads/yang21
Commit: 8006bb135414810efe6734054ad5e4c3ac9c95d5
Parents: e27f545
Author: Li Yang 
Authored: Mon Oct 17 17:07:14 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 17:07:14 2016 +0800

--
 build/conf/kylin_job_conf.xml   | 8 
 build/conf/kylin_job_conf_inmem.xml | 8 
 2 files changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8006bb13/build/conf/kylin_job_conf.xml
--
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 96b806c..b17c54a 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -85,4 +85,12 @@
 Set task timeout to 1 hour
 
 
+
+mapreduce.map.speculative
+false
+
+
+mapreduce.reduce.speculative
+false
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8006bb13/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index fea2f68..73b5cc5 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -104,4 +104,12 @@
 
 
 
+
+mapreduce.map.speculative
+false
+
+
+mapreduce.reduce.speculative
+false
+
 

[5/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a27ff16e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a27ff16e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a27ff16e

Branch: refs/heads/yang21-cdh5.7
Commit: a27ff16ed4c5187805cc846ec17fbbf7200faa6a
Parents: 8006bb1
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Mon Oct 17 22:35:58 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a27ff16e/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[2/6] kylin git commit: KYLIN-2099 fix sample table/cube desc

[liyang]

KYLIN-2099 fix sample table/cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e27f545f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e27f545f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e27f545f

Branch: refs/heads/yang21-cdh5.7
Commit: e27f545fb2af50f9259aa9af40ec47ac6f2ec25a
Parents: 0c2beec
Author: Li Yang 
Authored: Mon Oct 17 16:42:44 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 16:42:44 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql   | 16 
 .../template/cube_desc/kylin_sales_cube_desc.json   |  2 +-
 .../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json |  4 ++--
 3 files changed, 11 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e27f545f/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index 8399ee8..bc78802 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -116,10 +116,10 @@ CAL_DT date
 ,YTM_YN_ID tinyint
 ,YTQ_YN_ID tinyint
 ,YTW_YN_ID tinyint
-,CRE_DATE string
-,CRE_USER string
-,UPD_DATE string
-,UPD_USER string
+,KYLIN_CAL_DT_CRE_DATE string
+,KYLIN_CAL_DT_CRE_USER string
+,KYLIN_CAL_DT_UPD_DATE string
+,KYLIN_CAL_DT_UPD_USER string
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
@@ -136,10 +136,10 @@ LEAF_CATEG_ID bigint
 ,REGN_CATEG string
 ,USER_DEFINED_FIELD1 string
 ,USER_DEFINED_FIELD3 string
-,CRE_DATE string
-,UPD_DATE string
-,CRE_USER string
-,UPD_USER string
+,KYLIN_GROUPINGS_CRE_DATE string
+,KYLIN_GROUPINGS_UPD_DATE string
+,KYLIN_GROUPINGS_CRE_USER string
+,KYLIN_GROUPINGS_UPD_USER string
 ,META_CATEG_ID decimal
 ,META_CATEG_NAME string
 ,CATEG_LVL2_ID decimal

http://git-wip-us.apache.org/repos/asf/kylin/blob/e27f545f/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index cc1cd76..3e9bff4 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -14,7 +14,7 @@
 "name" : "CATEGORY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "{FK}",
-"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER" ]
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
   }, {
 "name" : "CATEGORY_HIERARCHY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",

http://git-wip-us.apache.org/repos/asf/kylin/blob/e27f545f/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
--
diff --git 
a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json 
b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
index 8e97309..1e1c6a2 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
@@ -40,7 +40,7 @@
 "datatype" : "string"
   }, {
 "id" : "10",
-"name" : "UPD_DATE",
+"name" : "KYLIN_GROUPINGS_UPD_DATE",
 "datatype" : "string"
   }, {
 "id" : "11",
@@ -48,7 +48,7 @@
 "datatype" : "string"
   }, {
 "id" : "12",
-"name" : "UPD_USER",
+"name" : "KYLIN_GROUPINGS_UPD_USER",
 "datatype" : "string"
   }, {
 "id" : "13",

[6/6] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

[liyang]

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41a19e91
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41a19e91
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41a19e91

Branch: refs/heads/yang21-cdh5.7
Commit: 41a19e91755f5c463a0768376d04f609ac0fc746
Parents: a27ff16
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Yang Li 
Committed: Mon Oct 17 22:38:34 2016 +

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/41a19e91/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/41a19e91/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/41a19e91/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[3/6] kylin git commit: minor, mapreduce speculative off by default

[liyang]

minor, mapreduce speculative off by default


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8006bb13
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8006bb13
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8006bb13

Branch: refs/heads/yang21-cdh5.7
Commit: 8006bb135414810efe6734054ad5e4c3ac9c95d5
Parents: e27f545
Author: Li Yang 
Authored: Mon Oct 17 17:07:14 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 17:07:14 2016 +0800

--
 build/conf/kylin_job_conf.xml   | 8 
 build/conf/kylin_job_conf_inmem.xml | 8 
 2 files changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8006bb13/build/conf/kylin_job_conf.xml
--
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 96b806c..b17c54a 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -85,4 +85,12 @@
 Set task timeout to 1 hour
 
 
+
+mapreduce.map.speculative
+false
+
+
+mapreduce.reduce.speculative
+false
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8006bb13/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index fea2f68..73b5cc5 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -104,4 +104,12 @@
 
 
 
+
+mapreduce.map.speculative
+false
+
+
+mapreduce.reduce.speculative
+false
+
 

[4/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/a27ff16e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a27ff16e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[1/6] kylin git commit: minor, better name for hive build step [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 5d5e99898 -> 41a19e917 (forced update)


minor, better name for hive build step


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c2beec3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c2beec3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c2beec3

Branch: refs/heads/yang21-cdh5.7
Commit: 0c2beec329381e8bf8ddc9658bea7160c94b97b1
Parents: 36a404f
Author: Li Yang 
Authored: Mon Oct 17 13:38:42 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 13:38:42 2016 +0800

--
 .../java/org/apache/kylin/job/constant/ExecutableConstants.java| 1 +
 .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java| 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0c2beec3/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 893c034..ad0b1b1 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -51,6 +51,7 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_SAVE_STATISTICS = "Save Cuboid 
Statistics";
 public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
 public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
+public static final String STEP_NAME_HIVE_CLEANUP = "Hive Cleanup";
 public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage 
Collection";
 public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage 
Collection on HDFS";
 public static final String STEP_NAME_BUILD_II = "Build Inverted Index";

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c2beec3/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 520d7cc..363a8d9 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -233,7 +233,7 @@ public class HiveMRInput implements IMRInput {
 @Override
 public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) {
 GarbageCollectionStep step = new GarbageCollectionStep();
-step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
+step.setName(ExecutableConstants.STEP_NAME_HIVE_CLEANUP);
 step.setIntermediateTableIdentity(getIntermediateTableIdentity());
 step.setExternalDataPath(JoinedFlatTable.getTableDir(flatDesc, 
JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId(;
 
step.setHiveViewIntermediateTableIdentities(hiveViewIntermediateTables);

kylin git commit: KYLIN-2091 bug fix

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master 07e81fd0b -> c13ab1c32


KYLIN-2091 bug fix

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c13ab1c3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c13ab1c3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c13ab1c3

Branch: refs/heads/master
Commit: c13ab1c324bf6a56e75365e0e1ab5ecd48fd048a
Parents: 07e81fd
Author: shaofengshi 
Authored: Tue Oct 18 10:43:13 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 18 10:43:13 2016 +0800

--
 .../main/java/org/apache/kylin/cube/CubeSegment.java| 12 
 .../apache/kylin/source/kafka/job/SeekOffsetStep.java   |  2 ++
 2 files changed, 2 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c13ab1c3/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index eb5b389..fdf1fb0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -564,12 +564,6 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetEnd(Map 
sourcePartitionOffsetEnd) {
 this.sourcePartitionOffsetEnd = sourcePartitionOffsetEnd;
-long total = 0;
-for (Long x : sourcePartitionOffsetEnd.values()) {
-total += x;
-}
-
-this.sourceOffsetEnd = total;
 }
 
 public Map getSourcePartitionOffsetStart() {
@@ -578,11 +572,5 @@ public class CubeSegment implements 
Comparable, IBuildable, ISegmen
 
 public void setSourcePartitionOffsetStart(Map 
sourcePartitionOffsetStart) {
 this.sourcePartitionOffsetStart = sourcePartitionOffsetStart;
-long total = 0;
-for (Long x : sourcePartitionOffsetStart.values()) {
-total += x;
-}
-
-this.sourceOffsetStart = total;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c13ab1c3/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
index 2a3dbb5..a26f39d 100644
--- 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
+++ 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/job/SeekOffsetStep.java
@@ -116,6 +116,8 @@ public class SeekOffsetStep extends AbstractExecutable {
 }
 
 if (totalEndOffset > totalStartOffset) {
+segment.setSourceOffsetStart(totalStartOffset);
+segment.setSourceOffsetEnd(totalEndOffset);
 segment.setSourcePartitionOffsetStart(startOffsets);
 segment.setSourcePartitionOffsetEnd(endOffsets);
 segment.setName(CubeSegment.makeSegmentName(0, 0, 
totalStartOffset, totalEndOffset));

kylin git commit: update faq and advanced_settings

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/document 8e639ce01 -> 8989d052c


update faq and advanced_settings


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8989d052
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8989d052
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8989d052

Branch: refs/heads/document
Commit: 8989d052ce79dfa2e9f2afce3c31062844aaebe5
Parents: 8e639ce
Author: shaofengshi 
Authored: Tue Oct 18 10:30:12 2016 +0800
Committer: shaofengshi 
Committed: Tue Oct 18 10:30:12 2016 +0800

--
 website/_docs15/gettingstarted/faq.md   | 48 
 website/_docs15/install/advance_settings.md | 12 ++
 2 files changed, 45 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8989d052/website/_docs15/gettingstarted/faq.md
--
diff --git a/website/_docs15/gettingstarted/faq.md 
b/website/_docs15/gettingstarted/faq.md
index 4306813..e070159 100644
--- a/website/_docs15/gettingstarted/faq.md
+++ b/website/_docs15/gettingstarted/faq.md
@@ -6,21 +6,41 @@ permalink: /docs15/gettingstarted/faq.html
 since: v0.6.x
 ---
 
- Build cube failed due to "error check status"
+ 1. "bin/find-hive-dependency.sh" can locate hive/hcat jars in local, but 
Kylin reports error like "java.lang.NoClassDefFoundError: 
org/apache/hive/hcatalog/mapreduce/HCatInputFormat"
+
+  * Kylin need many dependent jars (hadoop/hive/hcat/hbase/kafka) on classpath 
to work, but Kylin doesn't ship them. It will seek these jars from your local 
machine by running commands like `hbase classpath`, `hive -e set` etc. The 
founded jars' path will be appended to the environment variable 
*HBASE_CLASSPATH* (Kylin uses `hbase` shell command to start up, which will 
read this). But in some Hadoop distribution (like EMR 5.0), the `hbase` shell 
doesn't keep the origin `HBASE_CLASSPATH` value, that causes the 
"NoClassDefFoundError".
+
+  * To fix this, find the hbase shell script (in hbase/bin folder), and search 
*HBASE_CLASSPATH*, check whether it overwrite the value like :
+
+  {% highlight Groff markup %}
+  export 
HBASE_CLASSPATH=$HADOOP_CONF:$HADOOP_HOME/*:$HADOOP_HOME/lib/*:$ZOOKEEPER_HOME/*:$ZOOKEEPER_HOME/lib/*
+  {% endhighlight %}
+
+  * If true, change it to keep the origin value like:
+
+   {% highlight Groff markup %}
+  export 
HBASE_CLASSPATH=$HADOOP_CONF:$HADOOP_HOME/*:$HADOOP_HOME/lib/*:$ZOOKEEPER_HOME/*:$ZOOKEEPER_HOME/lib/*:$HBASE_CLASSPATH
+  {% endhighlight %}
+
+ 2. Get "java.lang.IllegalArgumentException: Too high cardinality is not 
suitable for dictionary -- cardinality: 5220674" in "Build Dimension 
Dictionary" step
+
+  * Kylin uses "Dictionary" encoding to encode/decode the dimension values 
(check [this blog](/blog/2015/08/13/kylin-dictionary/)); Usually a dimension's 
cardinality is less than millions, so the "Dict" encoding is good to use. As 
dictionary need be persisted and loaded into memory, if a dimension's 
cardinality is very high, the memory footprint will be tremendous, so Kylin add 
a check on this. If you see this error, suggest to identify the UHC dimension 
first and then re-evaluate the design (whether need to make that as 
dimension?). If must keep it, you can by-pass this error with couple ways: 1) 
change to use other encoding (like `fixed_length`, `integer`) 2) or set a 
bigger value for `kylin.dictionary.max.cardinality` in `conf/kylin.properties`.
+
+ 3. Build cube failed due to "error check status"
 
   * Check if `kylin.log` contains 
*yarn.resourcemanager.webapp.address:http://0.0.0.0:8088* and 
*java.net.ConnectException: Connection refused*
   * If yes, then the problem is the address of resource manager was not 
available in yarn-site.xml
   * A workaround is update `kylin.properties`, set 
`kylin.job.yarn.app.rest.check.status.url=http://YOUR_RM_NODE:8088/ws/v1/cluster/apps/${job_id}?anonymous=true`
 
- HBase cannot get master address from ZooKeeper on Hortonworks Sandbox
+ 4. HBase cannot get master address from ZooKeeper on Hortonworks Sandbox

   * By default hortonworks disables hbase, you'll have to start hbase in 
ambari homepage first.
 
- Map Reduce Job information cannot display on Hortonworks Sandbox
+ 5. Map Reduce Job information cannot display on Hortonworks Sandbox

   * Check out 
[https://github.com/KylinOLAP/Kylin/issues/40](https://github.com/KylinOLAP/Kylin/issues/40)
 
- Install Kylin on CDH 5.2 or Hadoop 2.5.x
+ 6. How to Install Kylin on CDH 5.2 or Hadoop 2.5.x
 
   * Check out discussion: 

kylin git commit: KYLIN-2099 fix sample model

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21 8006bb135 -> 274d5ce8f


KYLIN-2099 fix sample model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/274d5ce8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/274d5ce8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/274d5ce8

Branch: refs/heads/yang21
Commit: 274d5ce8f640d2d3a0e3ddd9c75a505748843b3e
Parents: 8006bb1
Author: Li Yang 
Authored: Tue Oct 18 12:36:43 2016 +0800
Committer: Li Yang 
Committed: Tue Oct 18 12:36:43 2016 +0800

--
 examples/sample_cube/template/model_desc/kylin_sales_model.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/274d5ce8/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 13fc356..3c4fa5a 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -21,7 +21,7 @@
 "columns" : ["CAL_DT", "WEEK_BEG_DT"]
   }, {
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
-"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
+"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", 
"META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
 "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]

kylin git commit: KYLIN-2099 fix sample model

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master c13ab1c32 -> 03118bd44


KYLIN-2099 fix sample model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03118bd4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03118bd4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03118bd4

Branch: refs/heads/master
Commit: 03118bd445e7cfba14b75751fb9f478fe650b88a
Parents: c13ab1c
Author: Li Yang 
Authored: Tue Oct 18 12:36:43 2016 +0800
Committer: Li Yang 
Committed: Tue Oct 18 12:37:13 2016 +0800

--
 examples/sample_cube/template/model_desc/kylin_sales_model.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/03118bd4/examples/sample_cube/template/model_desc/kylin_sales_model.json
--
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json 
b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 13fc356..3c4fa5a 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -21,7 +21,7 @@
 "columns" : ["CAL_DT", "WEEK_BEG_DT"]
   }, {
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
-"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
+"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", 
"META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
   }, {
 "table" : "DEFAULT.KYLIN_SALES",
 "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", 
"REGION" ]

[1/5] kylin git commit: minor, better name for hive build step [Forced Update!]

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x fe91c21f7 -> a27ff16ed (forced update)


minor, better name for hive build step


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c2beec3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c2beec3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c2beec3

Branch: refs/heads/yang21-hbase1.x
Commit: 0c2beec329381e8bf8ddc9658bea7160c94b97b1
Parents: 36a404f
Author: Li Yang 
Authored: Mon Oct 17 13:38:42 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 13:38:42 2016 +0800

--
 .../java/org/apache/kylin/job/constant/ExecutableConstants.java| 1 +
 .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java| 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0c2beec3/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 893c034..ad0b1b1 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -51,6 +51,7 @@ public final class ExecutableConstants {
 public static final String STEP_NAME_SAVE_STATISTICS = "Save Cuboid 
Statistics";
 public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
 public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
+public static final String STEP_NAME_HIVE_CLEANUP = "Hive Cleanup";
 public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage 
Collection";
 public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage 
Collection on HDFS";
 public static final String STEP_NAME_BUILD_II = "Build Inverted Index";

http://git-wip-us.apache.org/repos/asf/kylin/blob/0c2beec3/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 520d7cc..363a8d9 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -233,7 +233,7 @@ public class HiveMRInput implements IMRInput {
 @Override
 public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) {
 GarbageCollectionStep step = new GarbageCollectionStep();
-step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
+step.setName(ExecutableConstants.STEP_NAME_HIVE_CLEANUP);
 step.setIntermediateTableIdentity(getIntermediateTableIdentity());
 step.setExternalDataPath(JoinedFlatTable.getTableDir(flatDesc, 
JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId(;
 
step.setHiveViewIntermediateTableIdentities(hiveViewIntermediateTables);

[5/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a27ff16e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a27ff16e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a27ff16e

Branch: refs/heads/yang21-hbase1.x
Commit: a27ff16ed4c5187805cc846ec17fbbf7200faa6a
Parents: 8006bb1
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Yang Li 
Committed: Mon Oct 17 22:35:58 2016 +

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 49 files changed, 408 insertions(+), 462 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a27ff16e/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See 

[3/5] kylin git commit: minor, mapreduce speculative off by default

[liyang]

minor, mapreduce speculative off by default


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8006bb13
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8006bb13
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8006bb13

Branch: refs/heads/yang21-hbase1.x
Commit: 8006bb135414810efe6734054ad5e4c3ac9c95d5
Parents: e27f545
Author: Li Yang 
Authored: Mon Oct 17 17:07:14 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 17:07:14 2016 +0800

--
 build/conf/kylin_job_conf.xml   | 8 
 build/conf/kylin_job_conf_inmem.xml | 8 
 2 files changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8006bb13/build/conf/kylin_job_conf.xml
--
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 96b806c..b17c54a 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -85,4 +85,12 @@
 Set task timeout to 1 hour
 
 
+
+mapreduce.map.speculative
+false
+
+
+mapreduce.reduce.speculative
+false
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8006bb13/build/conf/kylin_job_conf_inmem.xml
--
diff --git a/build/conf/kylin_job_conf_inmem.xml 
b/build/conf/kylin_job_conf_inmem.xml
index fea2f68..73b5cc5 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -104,4 +104,12 @@
 
 
 
+
+mapreduce.map.speculative
+false
+
+
+mapreduce.reduce.speculative
+false
+
 

[2/5] kylin git commit: KYLIN-2099 fix sample table/cube desc

[liyang]

KYLIN-2099 fix sample table/cube desc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e27f545f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e27f545f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e27f545f

Branch: refs/heads/yang21-hbase1.x
Commit: e27f545fb2af50f9259aa9af40ec47ac6f2ec25a
Parents: 0c2beec
Author: Li Yang 
Authored: Mon Oct 17 16:42:44 2016 +0800
Committer: Li Yang 
Committed: Mon Oct 17 16:42:44 2016 +0800

--
 examples/sample_cube/create_sample_tables.sql   | 16 
 .../template/cube_desc/kylin_sales_cube_desc.json   |  2 +-
 .../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json |  4 ++--
 3 files changed, 11 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e27f545f/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index 8399ee8..bc78802 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -116,10 +116,10 @@ CAL_DT date
 ,YTM_YN_ID tinyint
 ,YTQ_YN_ID tinyint
 ,YTW_YN_ID tinyint
-,CRE_DATE string
-,CRE_USER string
-,UPD_DATE string
-,UPD_USER string
+,KYLIN_CAL_DT_CRE_DATE string
+,KYLIN_CAL_DT_CRE_USER string
+,KYLIN_CAL_DT_UPD_DATE string
+,KYLIN_CAL_DT_UPD_USER string
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
@@ -136,10 +136,10 @@ LEAF_CATEG_ID bigint
 ,REGN_CATEG string
 ,USER_DEFINED_FIELD1 string
 ,USER_DEFINED_FIELD3 string
-,CRE_DATE string
-,UPD_DATE string
-,CRE_USER string
-,UPD_USER string
+,KYLIN_GROUPINGS_CRE_DATE string
+,KYLIN_GROUPINGS_UPD_DATE string
+,KYLIN_GROUPINGS_CRE_USER string
+,KYLIN_GROUPINGS_UPD_USER string
 ,META_CATEG_ID decimal
 ,META_CATEG_NAME string
 ,CATEG_LVL2_ID decimal

http://git-wip-us.apache.org/repos/asf/kylin/blob/e27f545f/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
--
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json 
b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
index cc1cd76..3e9bff4 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -14,7 +14,7 @@
 "name" : "CATEGORY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
 "column" : "{FK}",
-"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", 
"UPD_USER" ]
+"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
   }, {
 "name" : "CATEGORY_HIERARCHY",
 "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",

http://git-wip-us.apache.org/repos/asf/kylin/blob/e27f545f/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
--
diff --git 
a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json 
b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
index 8e97309..1e1c6a2 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
@@ -40,7 +40,7 @@
 "datatype" : "string"
   }, {
 "id" : "10",
-"name" : "UPD_DATE",
+"name" : "KYLIN_GROUPINGS_UPD_DATE",
 "datatype" : "string"
   }, {
 "id" : "11",
@@ -48,7 +48,7 @@
 "datatype" : "string"
   }, {
 "id" : "12",
-"name" : "UPD_USER",
+"name" : "KYLIN_GROUPINGS_UPD_USER",
 "datatype" : "string"
   }, {
 "id" : "13",

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/a27ff16e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a27ff16e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name = 

[3/3] kylin git commit: KYLIN-2095 Allow cube to override Hive job configuration by properties

[shaofengshi]

KYLIN-2095 Allow cube to override Hive job configuration by properties

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07e81fd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07e81fd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07e81fd0

Branch: refs/heads/master
Commit: 07e81fd0b744e782d84383e327b1923cfc178d42
Parents: cc2b59f
Author: shaofengshi 
Authored: Mon Oct 17 22:07:57 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 22:07:57 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 ++
 .../apache/kylin/source/hive/HiveMRInput.java   | 42 
 2 files changed, 30 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/07e81fd0/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 73ac788..5a06813 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -250,6 +250,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getPropertiesByPrefix("kylin.job.mr.config.override.");
 }
 
+public Map getHiveConfigOverride() {
+return getPropertiesByPrefix("kylin.hive.config.override.");
+}
+
 public String getKylinSparkJobJarPath() {
 final String jobJar = getOptional("kylin.job.jar.spark");
 if (StringUtils.isNotEmpty(jobJar)) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/07e81fd0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 4ec8d3d..f3fceb1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -20,6 +20,8 @@ package org.apache.kylin.source.hive;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.io.IOUtils;
@@ -61,7 +63,6 @@ import com.google.common.collect.Sets;
 
 public class HiveMRInput implements IMRInput {
 
-private static final String MR_OVERRIDE_JOB_QUEUENAME = 
"mapreduce.job.queuename";
 
 @Override
 public IMRBatchCubingInputSide 
getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
@@ -157,9 +158,7 @@ public class HiveMRInput implements IMRInput {
 hiveInitBuf.append("USE 
").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
 
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
 final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
-if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
-hiveInitBuf.append("SET 
mapreduce.job.queuename=").append(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME)).append(";\n");
-}
+appendHiveOverrideProperties(kylinConfig, hiveInitBuf);
 String rowCountOutputDir = 
JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/row_count";
 
 RedistributeFlatHiveTableStep step = new 
RedistributeFlatHiveTableStep();
@@ -178,10 +177,7 @@ public class HiveMRInput implements IMRInput {
 
 final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
 final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
-if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
-hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
-
kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) + ";\n");
-}
+appendHiveOverrideProperties2(kylinConfig, hiveCmdBuilder);
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
 hiveCmdBuilder.addStatement("set 
hive.exec.compress.output=false;\n");
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateCountDataStatement(flatTableDesc,
 rowCountOutputDir));
@@ -192,6 +188,7 @@ public class HiveMRInput implements IMRInput {
 return step;
 }
 
+
 public ShellExecutable 

[1/3] kylin git commit: delete the blank line code

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/master c92f79ad3 -> 07e81fd0b


delete the blank line code

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cc2b59fa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cc2b59fa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cc2b59fa

Branch: refs/heads/master
Commit: cc2b59faf1a12e76e6af83ba3a19fbe27ba54cc3
Parents: ac356f0
Author: lijieliang 
Authored: Fri Oct 14 14:28:01 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 18:25:26 2016 +0800

--
 .../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cc2b59fa/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 2ec1fbb..4ec8d3d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -130,7 +130,6 @@ public class HiveMRInput implements IMRInput {
 @Override
 public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable 
jobFlow) {
 final String cubeName = 
CubingExecutableUtil.getCubeName(jobFlow.getParams());
-
 final KylinConfig kylinConfig = 
CubeManager.getInstance(conf.getConfig()).getCube(cubeName).getConfig();
 
 String createFlatTableMethod = 
kylinConfig.getCreateFlatHiveTableMethod();

[2/3] kylin git commit: hive job use overrided MR job configuration by cube properties

[shaofengshi]

hive job use overrided MR job configuration by cube properties

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac356f01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac356f01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac356f01

Branch: refs/heads/master
Commit: ac356f014d52c4b13ad72e9d6a537e50e9ace5fb
Parents: c92f79a
Author: lijieliang 
Authored: Fri Oct 14 13:01:32 2016 +0800
Committer: shaofengshi 
Committed: Mon Oct 17 18:25:26 2016 +0800

--
 .../apache/kylin/source/hive/HiveMRInput.java   | 24 +---
 1 file changed, 21 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ac356f01/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 202e480..2ec1fbb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -35,6 +35,7 @@ import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.engine.mr.IMRInput;
 import org.apache.kylin.engine.mr.JobBuilderSupport;
@@ -60,6 +61,8 @@ import com.google.common.collect.Sets;
 
 public class HiveMRInput implements IMRInput {
 
+private static final String MR_OVERRIDE_JOB_QUEUENAME = 
"mapreduce.job.queuename";
+
 @Override
 public IMRBatchCubingInputSide 
getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
 return new BatchCubingInputSide(flatDesc);
@@ -154,7 +157,10 @@ public class HiveMRInput implements IMRInput {
 StringBuilder hiveInitBuf = new StringBuilder();
 hiveInitBuf.append("USE 
").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
 
hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
-
+final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
+if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+hiveInitBuf.append("SET 
mapreduce.job.queuename=").append(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME)).append(";\n");
+}
 String rowCountOutputDir = 
JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/row_count";
 
 RedistributeFlatHiveTableStep step = new 
RedistributeFlatHiveTableStep();
@@ -172,6 +178,11 @@ public class HiveMRInput implements IMRInput {
 final ShellExecutable step = new ShellExecutable();
 
 final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+final KylinConfig kylinConfig = ((CubeSegment) 
flatTableDesc.getSegment()).getConfig();
+if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
+
kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) + ";\n");
+}
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
 hiveCmdBuilder.addStatement("set 
hive.exec.compress.output=false;\n");
 
hiveCmdBuilder.addStatement(JoinedFlatTable.generateCountDataStatement(flatTableDesc,
 rowCountOutputDir));
@@ -187,7 +198,7 @@ public class HiveMRInput implements IMRInput {
 
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP);
 HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
 
-KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+KylinConfig kylinConfig = ((CubeSegment) 
flatDesc.getSegment()).getConfig();
 MetadataManager metadataManager = 
MetadataManager.getInstance(kylinConfig);
 final Set lookupViewsTables = Sets.newHashSet();
 
@@ -201,6 +212,10 @@ public class HiveMRInput implements IMRInput {
 if (lookupViewsTables.size() == 0) {
 return null;
 }
+if 
(kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) != null) {
+hiveCmdBuilder.addStatement("SET mapreduce.job.queuename =" +
+
kylinConfig.getMRConfigOverride().get(MR_OVERRIDE_JOB_QUEUENAME) +