date:20161103

[9/9] kylin git commit: KYLIN-2135 minor format update

2016-11-03 Thread shaofengshi

KYLIN-2135 minor format update

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0a20a9b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0a20a9b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0a20a9b0

Branch: refs/heads/KYLIN-2135
Commit: 0a20a9b093807dac60f758de9a3f6fc0a6d72a62
Parents: fbd2132
Author: shaofengshi 
Authored: Thu Nov 3 18:49:50 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 4 13:40:48 2016 +0800

--
 .../kylin/engine/mr/DFSFileTableReader.java | 92 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java | 34 
 2 files changed, 61 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/0a20a9b0/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
index dda1d6f..173c908 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
@@ -23,14 +23,15 @@ import java.io.Closeable;
 import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.SequenceFile;
@@ -57,7 +58,7 @@ public class DFSFileTableReader implements TableReader {
 
 private String filePath;
 private String delim;
-private List readerList;
+private List readerList;
 
 private String curLine;
 private String[] curColumns;
@@ -72,33 +73,33 @@ public class DFSFileTableReader implements TableReader {
 this.filePath = filePath;
 this.delim = delim;
 this.expectedColumnNumber = expectedColumnNumber;
-this.readerList = new ArrayList();
+this.readerList = new ArrayList();
 
 FileSystem fs = HadoopUtil.getFileSystem(filePath);
 
-ArrayList allFiles = new ArrayList<>();
-FileStatus status = fs.getFileStatus(new Path(filePath));
-if (status.isFile()) {
-allFiles.add(status);
-} else {
-FileStatus[] listStatus = fs.listStatus(new Path(filePath));
-allFiles.addAll(Arrays.asList(listStatus));
-}
-
-try {
-for (FileStatus f : allFiles) {
-RowReader rowReader = new 
SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, f.getPath().toString());
-this.readerList.add(rowReader);
-}
+ArrayList allFiles = new ArrayList<>();
+FileStatus status = fs.getFileStatus(new Path(filePath));
+if (status.isFile()) {
+allFiles.add(status);
+} else {
+FileStatus[] listStatus = fs.listStatus(new Path(filePath));
+allFiles.addAll(Arrays.asList(listStatus));
+}
+
+try {
+for (FileStatus f : allFiles) {
+RowReader rowReader = new 
SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, f.getPath().toString());
+this.readerList.add(rowReader);
+}
 } catch (IOException e) {
 if (isExceptionSayingNotSeqFile(e) == false)
 throw e;
 
-this.readerList = new ArrayList();
-for (FileStatus f : allFiles) {
-RowReader rowReader = new CsvRowReader(fs, 
f.getPath().toString());
-this.readerList.add(rowReader);
-}
+this.readerList = new ArrayList();
+for (FileStatus f : allFiles) {
+RowReader rowReader = new CsvRowReader(fs, 
f.getPath().toString());
+this.readerList.add(rowReader);
+}
 }
 }
 
@@ -114,20 +115,20 @@ public class DFSFileTableReader implements TableReader {
 
 @Override
 public boolean next() throws IOException {
-int curReaderIndex = -1;
-RowReader curReader;
-
-while (++curReaderIndex < readerList.size()) {
-curReader = readerList.get(curReaderIndex);
-curLine = curReader.nextLine();
-curColumns = null;
-
-if (curLine

[3/9] kylin git commit: minor, fix PrintHBaseConfig checkstyle

2016-11-03 Thread shaofengshi

minor, fix PrintHBaseConfig checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50059e38
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50059e38
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50059e38

Branch: refs/heads/KYLIN-2135
Commit: 50059e388ea59c42fb9eeea2d1d92927114bcdbd
Parents: 03f3857
Author: Li Yang 
Authored: Thu Nov 3 14:14:05 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 14:15:03 2016 +0800

--
 .../java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/50059e38/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
index 634ebdf..f9b7daf 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
@@ -33,7 +33,7 @@ public class PrintHBaseConfig {
 MyConfig config = new MyConfig(HBaseConfiguration.create());
 
 if (args.length == 0) {
-for (Map.Entry item: config.getProps().entrySet()) {
+for (Map.Entry item : 
config.getProps().entrySet()) {
 System.out.println(item.getKey() + "=" + item.getValue());
 }
 System.exit(0);

[7/9] kylin git commit: minor change on error messages

2016-11-03 Thread shaofengshi

minor change on error messages

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c587b2ed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c587b2ed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c587b2ed

Branch: refs/heads/KYLIN-2135
Commit: c587b2ed3a968262a794e893f6af8fc109b02730
Parents: 235b123
Author: shaofengshi 
Authored: Fri Nov 4 13:33:19 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 4 13:33:19 2016 +0800

--
 .../org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 8 
 .../apache/kylin/engine/mr/common/HadoopStatusGetter.java| 2 +-
 .../main/java/org/apache/kylin/source/kafka/KafkaSource.java | 8 
 3 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 77791ce..21bb10e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -204,7 +204,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);
 } else {
 
-logger.info("No hive dependency jars set in the environment, will 
find them from jvm:");
+logger.info("No hive dependency jars set in the environment, will 
find them from classpath:");
 
 try {
 String hiveExecJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
@@ -227,17 +227,17 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for kafka dependencies
 if (kylinKafkaDependency != null) {
 kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
-logger.info("Kafka Dependencies Before Filtered: " + 
kylinKafkaDependency);
+logger.info("Kafka Dependencies: " + kylinKafkaDependency);
 StringUtil.appendWithSeparator(kylinDependency, 
kylinKafkaDependency);
 } else {
-logger.info("No Kafka dependency jars set in the environment, will 
find them from jvm:");
+logger.info("No Kafka dependency jar set in the environment, will 
find them from classpath:");
 try {
 String kafkaClientJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer"));
 StringUtil.appendWithSeparator(kylinDependency, 
kafkaClientJarPath);
 logger.info("kafka jar file: " + kafkaClientJarPath);
 
 } catch (ClassNotFoundException e) {
-logger.error("Cannot found kafka dependency jars: " + e);
+logger.warn("Not found kafka client jar from classpath, it is 
optional for normal build: " + e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
index 619de90..7dcb73e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
@@ -55,7 +55,7 @@ public class HadoopStatusGetter {
 private final String mrJobId;
 private final String yarnUrl;
 
-protected static final Logger logger = 
LoggerFactory.getLogger(HadoopStatusChecker.class);
+protected static final Logger logger = 
LoggerFactory.getLogger(HadoopStatusGetter.class);
 
 public HadoopStatusGetter(String yarnUrl, String mrJobId) {
 this.yarnUrl = yarnUrl;

http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
index bb676e6..7a5d94f 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
+++

[1/9] kylin git commit: minor, add util PrintHBaseConfig [Forced Update!]

2016-11-03 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2135 3b01b94c0 -> 0a20a9b09 (forced update)


minor, add util PrintHBaseConfig


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b80762cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b80762cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b80762cd

Branch: refs/heads/KYLIN-2135
Commit: b80762cd1dcc410179ab366391588486e1028ffa
Parents: 93e3020
Author: Li Yang 
Authored: Thu Nov 3 13:50:44 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 13:51:42 2016 +0800

--
 .../storage/hbase/util/PrintHBaseConfig.java| 62 
 1 file changed, 62 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b80762cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
new file mode 100644
index 000..634ebdf
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.util;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+/**
+ */
+public class PrintHBaseConfig {
+
+public static void main(String[] args) throws IOException {
+MyConfig config = new MyConfig(HBaseConfiguration.create());
+
+if (args.length == 0) {
+for (Map.Entry item: config.getProps().entrySet()) {
+System.out.println(item.getKey() + "=" + item.getValue());
+}
+System.exit(0);
+}
+
+if (args.length == 1) {
+System.out.println(config.get(args[0]));
+System.exit(0);
+}
+
+for (String arg : args) {
+System.out.println(arg + "=" + config.get(arg));
+}
+System.exit(0);
+}
+
+private static class MyConfig extends Configuration {
+MyConfig(Configuration other) {
+super(other);
+}
+
+protected synchronized Properties getProps() {
+return super.getProps();
+}
+}
+}

[8/9] kylin git commit: KYLIN-2135 Enlarge FactDistinctColumns reducer number

2016-11-03 Thread shaofengshi

KYLIN-2135 Enlarge FactDistinctColumns reducer number

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fbd21321
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fbd21321
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fbd21321

Branch: refs/heads/KYLIN-2135
Commit: fbd213218d41178263cd8affc321178dcde0b9d3
Parents: c587b2e
Author: kangkaisen 
Authored: Wed Oct 26 19:35:20 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 4 13:40:48 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 ++
 .../java/org/apache/kylin/cube/CubeManager.java | 35 
 .../kylin/engine/mr/DFSFileTableReader.java | 59 
 .../kylin/engine/mr/common/BatchConstants.java  |  5 ++
 .../mr/steps/FactDistinctColumnPartitioner.java | 11 +---
 .../engine/mr/steps/FactDistinctColumnsJob.java | 18 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java | 17 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 36 ++--
 .../mr/steps/FactDistinctHiveColumnsMapper.java | 12 +++-
 9 files changed, 171 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fbd21321/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index acc4eb1..9ac8142 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -374,6 +374,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.job.allow.empty.segment", "true"));
 }
 
+//UHC: ultra high cardinality columns, contain the ShardByColumns and the 
GlobalDictionaryColumns
+public int getUHCReducerCount() {
+return Integer.parseInt(getOptional("kylin.job.uhc.reducer.count", 
"3"));
+}
+
 public String getOverrideHiveTableLocation(String table) {
 return getOptional("hive.table.location." + table.toUpperCase());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fbd21321/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 16b468f..c04617d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -31,6 +31,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -43,6 +44,7 @@ import org.apache.kylin.common.persistence.Serializer;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
@@ -1047,4 +1049,37 @@ public class CubeManager implements IRealizationProvider 
{
 }
 return holes;
 }
+
+private final String GLOBAL_DICTIONNARY_CLASS = 
"org.apache.kylin.dict.GlobalDictionaryBuilder";
+
+//UHC (ultra high cardinality column): contain the ShardByColumns and the 
GlobalDictionaryColumns
+public int[] getUHCIndex(CubeDesc cubeDesc) throws IOException {
+List factDictCols = getAllDictColumnsOnFact(cubeDesc);
+int[] uhcIndex = new int[factDictCols.size()];
+
+//add GlobalDictionaryColumns
+List dictionaryDescList = cubeDesc.getDictionaries();
+if (dictionaryDescList != null) {
+for (DictionaryDesc dictionaryDesc : dictionaryDescList) {
+if (dictionaryDesc.getBuilderClass() != null && 
dictionaryDesc.getBuilderClass().equalsIgnoreCase(GLOBAL_DICTIONNARY_CLASS)) {
+for (int i = 0; i < factDictCols.size(); i++) {
+if 
(factDictCols.get(i).equals(dictionaryDesc.getColumnRef())) {
+uhcIndex[i] = 1;
+break;
+}
+}
+}
+}
+}
+
+//add ShardByColumns
+Set shardByColumns = cubeDesc.getShardByColumns();
+for (int i = 0; i <

kylin git commit: minor change on error messages

2016-11-03 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1 9a2fed5ba -> 207ab51dc


minor change on error messages

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/207ab51d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/207ab51d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/207ab51d

Branch: refs/heads/v1.6.0-rc1
Commit: 207ab51dc10457eea903ad5a75a775ce7f0d47ae
Parents: 9a2fed5
Author: shaofengshi 
Authored: Fri Nov 4 13:33:19 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 4 13:33:57 2016 +0800

--
 .../org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 8 
 .../apache/kylin/engine/mr/common/HadoopStatusGetter.java| 2 +-
 .../main/java/org/apache/kylin/source/kafka/KafkaSource.java | 8 
 3 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/207ab51d/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 77791ce..21bb10e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -204,7 +204,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);
 } else {
 
-logger.info("No hive dependency jars set in the environment, will 
find them from jvm:");
+logger.info("No hive dependency jars set in the environment, will 
find them from classpath:");
 
 try {
 String hiveExecJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
@@ -227,17 +227,17 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for kafka dependencies
 if (kylinKafkaDependency != null) {
 kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
-logger.info("Kafka Dependencies Before Filtered: " + 
kylinKafkaDependency);
+logger.info("Kafka Dependencies: " + kylinKafkaDependency);
 StringUtil.appendWithSeparator(kylinDependency, 
kylinKafkaDependency);
 } else {
-logger.info("No Kafka dependency jars set in the environment, will 
find them from jvm:");
+logger.info("No Kafka dependency jar set in the environment, will 
find them from classpath:");
 try {
 String kafkaClientJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer"));
 StringUtil.appendWithSeparator(kylinDependency, 
kafkaClientJarPath);
 logger.info("kafka jar file: " + kafkaClientJarPath);
 
 } catch (ClassNotFoundException e) {
-logger.error("Cannot found kafka dependency jars: " + e);
+logger.warn("Not found kafka client jar from classpath, it is 
optional for normal build: " + e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/207ab51d/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
index 619de90..7dcb73e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
@@ -55,7 +55,7 @@ public class HadoopStatusGetter {
 private final String mrJobId;
 private final String yarnUrl;
 
-protected static final Logger logger = 
LoggerFactory.getLogger(HadoopStatusChecker.class);
+protected static final Logger logger = 
LoggerFactory.getLogger(HadoopStatusGetter.class);
 
 public HadoopStatusGetter(String yarnUrl, String mrJobId) {
 this.yarnUrl = yarnUrl;

http://git-wip-us.apache.org/repos/asf/kylin/blob/207ab51d/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
index bb676e6..7a5d94f 100644
---

kylin git commit: minor change on error messages

2016-11-03 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/master 235b123d0 -> c587b2ed3


minor change on error messages

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c587b2ed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c587b2ed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c587b2ed

Branch: refs/heads/master
Commit: c587b2ed3a968262a794e893f6af8fc109b02730
Parents: 235b123
Author: shaofengshi 
Authored: Fri Nov 4 13:33:19 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 4 13:33:19 2016 +0800

--
 .../org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 8 
 .../apache/kylin/engine/mr/common/HadoopStatusGetter.java| 2 +-
 .../main/java/org/apache/kylin/source/kafka/KafkaSource.java | 8 
 3 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 77791ce..21bb10e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -204,7 +204,7 @@ public abstract class AbstractHadoopJob extends Configured 
implements Tool {
 StringUtil.appendWithSeparator(kylinDependency, filteredHive);
 } else {
 
-logger.info("No hive dependency jars set in the environment, will 
find them from jvm:");
+logger.info("No hive dependency jars set in the environment, will 
find them from classpath:");
 
 try {
 String hiveExecJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
@@ -227,17 +227,17 @@ public abstract class AbstractHadoopJob extends 
Configured implements Tool {
 // for kafka dependencies
 if (kylinKafkaDependency != null) {
 kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
-logger.info("Kafka Dependencies Before Filtered: " + 
kylinKafkaDependency);
+logger.info("Kafka Dependencies: " + kylinKafkaDependency);
 StringUtil.appendWithSeparator(kylinDependency, 
kylinKafkaDependency);
 } else {
-logger.info("No Kafka dependency jars set in the environment, will 
find them from jvm:");
+logger.info("No Kafka dependency jar set in the environment, will 
find them from classpath:");
 try {
 String kafkaClientJarPath = 
ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer"));
 StringUtil.appendWithSeparator(kylinDependency, 
kafkaClientJarPath);
 logger.info("kafka jar file: " + kafkaClientJarPath);
 
 } catch (ClassNotFoundException e) {
-logger.error("Cannot found kafka dependency jars: " + e);
+logger.warn("Not found kafka client jar from classpath, it is 
optional for normal build: " + e);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
index 619de90..7dcb73e 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
@@ -55,7 +55,7 @@ public class HadoopStatusGetter {
 private final String mrJobId;
 private final String yarnUrl;
 
-protected static final Logger logger = 
LoggerFactory.getLogger(HadoopStatusChecker.class);
+protected static final Logger logger = 
LoggerFactory.getLogger(HadoopStatusGetter.class);
 
 public HadoopStatusGetter(String yarnUrl, String mrJobId) {
 this.yarnUrl = yarnUrl;

http://git-wip-us.apache.org/repos/asf/kylin/blob/c587b2ed/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
--
diff --git 
a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java 
b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaSource.java
index bb676e6..7a5d94f 100644
---

svn commit: r1767985 - in /kylin/site: cn/download/index.html download/index.html feed.xml

2016-11-03 Thread lidong

Author: lidong
Date: Fri Nov  4 02:57:59 2016
New Revision: 1767985

URL: http://svn.apache.org/viewvc?rev=1767985=rev
Log:
update download page for 1.6.0 snapshots

Modified:
kylin/site/cn/download/index.html
kylin/site/download/index.html
kylin/site/feed.xml

Modified: kylin/site/cn/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/cn/download/index.html?rev=1767985=1767984=1767985=diff
==
--- kylin/site/cn/download/index.html (original)
+++ kylin/site/cn/download/index.html Fri Nov  4 02:57:59 2016
@@ -182,11 +182,12 @@
 1.6.0-SNAPSHOT (scalable streaming)
 
   
å¨è¿ä¸ªçæ¬ä¸Kylinå¯¹ä»¥Kafkaåæ°æ®æºæäºæ´å¥½çæ¯æï¼è¯·åèåå®¢New NRT Streaming in Apache 
Kylin.
-  äºè¿å¶åä¸è½½ (2016-10-26æ´æ°):
+  https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316121version=1266;>Release
 Notes (draft)
+  äºè¿å¶åä¸è½½ (2016-11-04æ´æ°):
 
-  for HBase 0.98 - https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz;>apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz
  [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc;>asc]
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5;>md5]
-  for HBase 1.x - https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz;>apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc;>asc]
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.md5;>md5]
-  for CDH 5.7/5.8 - https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz;>apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc;>asc]
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5;>md5]
+  for HBase 0.98 - https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz;>apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz
  [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc;>asc]
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5;>md5]
+  for HBase 1.x - https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz;>apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc;>asc]
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.md5;>md5]
+  for CDH 5.7/5.8 - https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz;>apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc;>asc]
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5;>md5]
 
   
 

Modified: kylin/site/download/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/download/index.html?rev=1767985=1767984=1767985=diff
==
--- kylin/site/download/index.html (original)
+++ kylin/site/download/index.html Fri Nov  4 02:57:59 2016
@@ -184,12 +184,13 @@ You can verify your download by followin
 
 1.6.0-SNAPSHOT (scalable streaming)
 
-  In this version Kylin has better support for Kafka as data source, check 
the blog New NRT Streaming in 
Apache Kylin.
-  Binary download (updated on 2016-10-26):
+  In this version Kylin has better support for Kafka as data source, read 
blog New NRT Streaming in Apache 
Kylin.
+  https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316121version=1266;>Release
 Notes (draft)
+  Binary download (updated on 2016-11-04):
 
-  for HBase 0.98 - https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz;>apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz
  [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc;>asc]
 [https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5;>md5]
-  for HBase 1.x -

kylin git commit: update download page for 1.6.0 snapshots

2016-11-03 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/document 900659890 -> cb7be7e15


update download page for 1.6.0 snapshots


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cb7be7e1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cb7be7e1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cb7be7e1

Branch: refs/heads/document
Commit: cb7be7e15393b5bb3a95051fe7da8b00401e1f53
Parents: 9006598
Author: shaofengshi 
Authored: Fri Nov 4 10:34:56 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 4 10:34:56 2016 +0800

--
 website/download/index.cn.md |  9 +
 website/download/index.md| 11 ++-
 2 files changed, 11 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cb7be7e1/website/download/index.cn.md
--
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index 6f96f33..7e3e16a 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -18,10 +18,11 @@ title: ä¸è½½
 
 ## 1.6.0-SNAPSHOT (scalable streaming)
 * 
å¨è¿ä¸ªçæ¬ä¸Kylinå¯¹ä»¥Kafkaåæ°æ®æºæäºæ´å¥½çæ¯æï¼è¯·åèåå®¢[New
 NRT Streaming in Apache Kylin](/blog/2016/10/18/new-nrt-streaming/).
-* äºè¿å¶åä¸è½½ (2016-10-26æ´æ°):
-   * for HBase 0.98 - 
[apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz)
  
\[[asc](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc)\]
 
\[[md5](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5)\]
-   * for HBase 1.x - 
[apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz)
 
\[[asc](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc)\]
 
\[[md5](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.md5)\]
-   * for CDH 5.7/5.8 - 
[apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz)
 
\[[asc](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc)\]
 
\[[md5](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5)\]
+* [Release Notes 
(draft)](https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316121=1266)
+* äºè¿å¶åä¸è½½ (2016-11-04æ´æ°):
+   * for HBase 0.98 - 
[apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz)
  
\[[asc](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc)\]
 
\[[md5](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5)\]
+   * for HBase 1.x - 
[apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz)
 
\[[asc](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc)\]
 
\[[md5](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.md5)\]
+   * for CDH 5.7/5.8 - 
[apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz)
 
\[[asc](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc)\]
 
\[[md5](https://dist.apache.org/repos/dist/dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5)\]
 
 
 ## ä»¥åççæ¬  

http://git-wip-us.apache.org/repos/asf/kylin/blob/cb7be7e1/website/download/index.md
--
diff --git a/website/download/index.md b/website/download/index.md
index e762735..4a9e938 100644
--- a/website/download/index.md
+++ b/website/download/index.md
@@ -18,11 +18,12 @@ You can verify your download by following these 
[procedures](https://www.apache.
 
 
 ## 1.6.0-SNAPSHOT (scalable streaming)
-* In this version Kylin has better support for Kafka as data source, check the 
blog [New NRT

svn commit: r16829 - /dev/kylin/apache-kylin-1.6.0-rc1-2/

2016-11-03 Thread shaofengshi

Author: shaofengshi
Date: Fri Nov  4 02:14:09 2016
New Revision: 16829

Log:
update 1.6.0 SNAPSHOT build

Added:
dev/kylin/apache-kylin-1.6.0-rc1-2/
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz   
(with props)

dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc

dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5

dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz
   (with props)

dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc

dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5

dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz
   (with props)

dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.asc

dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-hbase1.x-SNAPSHOT-bin.tar.gz.md5

Added: dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz
==
Binary file - no diff available.

Propchange: 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz
--
svn:mime-type = application/octet-stream

Added: 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc
==
--- 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc 
(added)
+++ 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.asc 
Fri Nov  4 02:14:09 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJYG+4sAAoJEKvJ+IVvz0cgkEcP/0WOd8HE7EwVANpJvhqH+gZs
+pCj5E/4QhEqpbaWL9BSUUs6EquLUOXLdDDWnRVmp3nsz8oTHGifcpOcBoQiaXbIx
+co3383zqzw+v60MC1Xy0HH9Z7EEE2v/R+y4JDqXyOjDi6kTJ6MtJSzXZoxTS5w/v
+vw9v6y/LbEnPVW2iREn4s0o7FpaxxZ/fzb/e03KjH3DIyvri2ga2VhsjrvtsMCnR
+7Ewe63ri+3iWHQ7gq0VstnguhpuD2rmzaD/HGeT8/lvSEerKk0pVFmAX0NrBHea9
++WA6HkYWARGksCCON/ThLkjwBKCNjB9PtVcFqlw+4UKDkzvGlhHDgr8WXtyQKI6i
+N4ZXLF+/z7vWXYcy0MhQybcF+OjYHw0wvr08gWvE/KVMH0tx0iNFXycnpkjeP/aJ
+87SY2E8E9ia8xX0AG7NUzKW7oYDzAdZ9MSafLwuyPwPMrryueAPloNTBAxAsWifj
+4mmCZy4/Xa8brLbZMN98WjZrWp6E53pv8/6rwwGjMto1kBpuIwyJbKHUvFtD+6wO
+9lKP+ESa3ctMf75FYr+OpT8pK5K7T+/Ub7QbxB1yP7wf/BVHrSICx+fxIp0g1tZW
+o6Mvk6GIh+7q/I6npAixUj7ysxIseV2TQOzYlzMUKsBzsSDwfJi3hkWcXxro6Qo8
+qizER3winqRnQAA+o6ae
+=VNxd
+-END PGP SIGNATURE-

Added: 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5
==
--- 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5 
(added)
+++ 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-SNAPSHOT-bin.tar.gz.md5 
Fri Nov  4 02:14:09 2016
@@ -0,0 +1 @@
+3dacfe49f8065c216765f141f75ecea3  -

Added: 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz
==
Binary file - no diff available.

Propchange: 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz
--
svn:mime-type = application/octet-stream

Added: 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc
==
--- 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc
 (added)
+++ 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.asc
 Fri Nov  4 02:14:09 2016
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+iQIcBAABAgAGBQJYG+5ZAAoJEKvJ+IVvz0cgPF0QAKK6ogU/fDL5TOAMbadOJoE8
+3bs7EylfNN+eOsXhvfhwhESxlooKYLceac4m1wXeWNJaArFNZk4hvqycQv+OUNV/
+FtsclUM8pSERykt/nQyFWLjPeoo0Nv5MF/BkjLSg1cIkEYanIrZgybWYwpCWuXeN
+udJDeIP4YHBcnCP0P6jaZWvHPMyGaV2EUe+ORzG0WAwb+4RVvjW5rcGxrXy84wN0
+QYGKgYvFwLtSQStxphITd87iniZCwSVX9iwAnisXA5jKV9syHtjgcbZ9b/+EB/OT
+5tDzAQcBqD/iFHuByMymsh3aE8nUhFb6hPFG5DffRh5eDvEK0zTxxo4+gvokoA6y
+PnoYDvER2VX9vLVSw5mjQlUWDaO9y6Xxl76P0Rs2UXeQ2POTuITyNfH0NnLXLcrv
+dLHO3z2/cTvZGA+IEn3pjCxkny2kPu0f0TCn399kFDVsF5fymDRWitWQ//+2y7Ol
+tcAqWWOMA73JtIu0PbGxs/ih6C1fevndbQLMmVPX40Jh3yF6NH6oieILf555of7L
+8CgBHtGaKeUukqtSVJXww+RJoFdNdumOZylbQh/d1MtLFAIlq/n1+XHlFtOhAJ/V
+u0TFHp746rG2H2z4hr2IGh2Mpcdvxe7HuOGsR8ryWEwrKuX93oz4u7JpDNVx61at
+KhgQ33vhVvbDsceJ4kip
+=nXjF
+-END PGP SIGNATURE-

Added: 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5
==
--- 
dev/kylin/apache-kylin-1.6.0-rc1-2/apache-kylin-1.6.0-cdh5.7-SNAPSHOT-bin.tar.gz.md5
 (added)
+++

[1/7] kylin git commit: KYLIN-1726 undo ‘task discard itself’ changes as it is not needed [Forced Update!]

2016-11-03 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1-cdh5.7 56215d0f0 -> fe1b21a34 (forced update)


KYLIN-1726 undo âtask discard itselfâ changes as it is not needed

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bc2ec1bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bc2ec1bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bc2ec1bf

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: bc2ec1bf17d63f7a5b9884ddc77fec6041d2b818
Parents: 7a37930
Author: shaofengshi 
Authored: Wed Nov 2 18:27:57 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 18:28:50 2016 +0800

--
 .../kylin/job/execution/AbstractExecutable.java |  2 -
 .../job/execution/DefaultChainedExecutable.java |  2 -
 .../kylin/job/execution/ExecuteResult.java  |  4 --
 .../kylin/job/DiscardedTestExecutable.java  | 41 
 .../impl/threadpool/DefaultSchedulerTest.java   | 16 
 .../kylin/provision/BuildCubeWithStream.java|  6 +--
 6 files changed, 1 insertion(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index b4ca469..90e4d3c 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -74,8 +74,6 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 if (!isDiscarded()) {
 if (result.succeed()) {
 executableManager.updateJobOutput(getId(), 
ExecutableState.SUCCEED, null, result.output());
-} else if (result.discarded()) {
-executableManager.updateJobOutput(getId(), 
ExecutableState.DISCARDED, null, result.output());
 } else {
 executableManager.updateJobOutput(getId(), 
ExecutableState.ERROR, null, result.output());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 5a57b05..39a5f4f 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -119,8 +119,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 } else {
 jobService.updateJobOutput(getId(), ExecutableState.READY, 
null, null);
 }
-} else if (result.discarded()) {
-jobService.updateJobOutput(getId(), ExecutableState.DISCARDED, 
null, result.output());
 } else {
 setEndTime(System.currentTimeMillis());
 jobService.updateJobOutput(getId(), ExecutableState.ERROR, null, 
result.output());

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
index 2347e7d..760a574 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
@@ -49,10 +49,6 @@ public final class ExecuteResult {
 return state == State.SUCCEED;
 }
 
-public boolean discarded() {
-return state == State.DISCARDED;
-}
-
 public String output() {
 return output;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java 
b/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
deleted file mode 100644
index 9362e18..000
--- a/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more

[7/7] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-03 Thread shaofengshi

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fe1b21a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fe1b21a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fe1b21a3

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: fe1b21a3446ac296e0ec0079c32e4b719cff554f
Parents: 7b5e01b
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: shaofengshi 
Committed: Fri Nov 4 09:48:52 2016 +0800

--
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 11 files changed, 428 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fe1b21a3/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/fe1b21a3/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/fe1b21a3/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+

[6/7] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread shaofengshi

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b5e01bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b5e01bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b5e01bb

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 7b5e01bbcb2d1ac1d8595c4aac2716e5324c0fb6
Parents: 9a2fed5
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 4 09:48:52 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java|  2 +-
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 48 files changed, 392 insertions(+), 448 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7b5e01bb/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.

[4/7] kylin git commit: KYLIN-2154 source table rows will be skipped if TOPN's group column contains NULL values

2016-11-03 Thread shaofengshi

KYLIN-2154 source table rows will be skipped if TOPN's group column contains 
NULL values


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a2fed5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a2fed5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a2fed5b

Branch: refs/heads/v1.6.0-rc1-cdh5.7
Commit: 9a2fed5bad2747f4570f92fd4f3f5380ca2c8ff4
Parents: df9a808
Author: Hongbin Ma 
Authored: Wed Nov 2 22:28:35 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 3 13:41:44 2016 +0800

--
 .../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 9 +++--
 .../apache/kylin/storage/gtrecord/CubeSegmentScanner.java   | 5 -
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a2fed5b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 7e788e4..f34c57c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.measure.topn;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -142,8 +143,12 @@ public class TopNMeasureType extends 
MeasureType {
 final ByteArray key = new ByteArray(keyLength);
 int offset = 0;
 for (int i = 0; i < dimensionEncodings.length; i++) {
-byte[] valueBytes = Bytes.toBytes(values[i + 1]);
-dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+if (values[i + 1] == null) {
+Arrays.fill(key.array(), offset, offset + 
dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL);
+} else {
+byte[] valueBytes = Bytes.toBytes(values[i + 1]);
+dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+}
 offset += dimensionEncodings[i].getLengthOfEncoding();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a2fed5b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 6e19c09..c6a6daa 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -53,6 +53,9 @@ public class CubeSegmentScanner implements IGTScanner {
 
 public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, 
Set dimensions, Set groups, //
 Collection metrics, TupleFilter originalfilter, 
StorageContext context, String gtStorage) {
+
+logger.info("Init CubeSegmentScanner for segment {}", 
cubeSeg.getName());
+
 this.cuboid = cuboid;
 this.cubeSeg = cubeSeg;
 
@@ -61,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner {
 //is working on its own copy
 byte[] serialize = TupleFilterSerializer.serialize(originalfilter, 
StringCodeSystem.INSTANCE);
 TupleFilter filter = TupleFilterSerializer.deserialize(serialize, 
StringCodeSystem.INSTANCE);
-
+
 // translate FunctionTupleFilter to IN clause
 ITupleFilterTransformer translator = new 
BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
 filter = translator.transform(filter);

[5/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread shaofengshi

http://git-wip-us.apache.org/repos/asf/kylin/blob/8b748cae/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+if 
(admin.isTableEnabled(TableName.valueOf(table))) {
+admin.disableTable(TableName.valueOf(table));
 }
-admin.deleteTable(table);
+admin.deleteTable(TableName.valueOf(table));
 logger.debug("Dropped htable: " + table);
 output.append("HBase table " + table + " is 
dropped. \n");
 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8b748cae/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
 import java.io.IOException;
 
 import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.AbstractApplication;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
 protected static final Logger logger = 
LoggerFactory.getLogger(CleanHtableCLI.class);
 
 private void clean() throws IOException {
-Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
 String name =

[1/6] kylin git commit: KYLIN-1726 undo ‘task discard itself’ changes as it is not needed [Forced Update!]

2016-11-03 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc1-hbase1.x 3940e80f9 -> 8b748cae3 (forced update)


KYLIN-1726 undo âtask discard itselfâ changes as it is not needed

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bc2ec1bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bc2ec1bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bc2ec1bf

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: bc2ec1bf17d63f7a5b9884ddc77fec6041d2b818
Parents: 7a37930
Author: shaofengshi 
Authored: Wed Nov 2 18:27:57 2016 +0800
Committer: shaofengshi 
Committed: Wed Nov 2 18:28:50 2016 +0800

--
 .../kylin/job/execution/AbstractExecutable.java |  2 -
 .../job/execution/DefaultChainedExecutable.java |  2 -
 .../kylin/job/execution/ExecuteResult.java  |  4 --
 .../kylin/job/DiscardedTestExecutable.java  | 41 
 .../impl/threadpool/DefaultSchedulerTest.java   | 16 
 .../kylin/provision/BuildCubeWithStream.java|  6 +--
 6 files changed, 1 insertion(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index b4ca469..90e4d3c 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -74,8 +74,6 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 if (!isDiscarded()) {
 if (result.succeed()) {
 executableManager.updateJobOutput(getId(), 
ExecutableState.SUCCEED, null, result.output());
-} else if (result.discarded()) {
-executableManager.updateJobOutput(getId(), 
ExecutableState.DISCARDED, null, result.output());
 } else {
 executableManager.updateJobOutput(getId(), 
ExecutableState.ERROR, null, result.output());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 5a57b05..39a5f4f 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@ -119,8 +119,6 @@ public class DefaultChainedExecutable extends 
AbstractExecutable implements Chai
 } else {
 jobService.updateJobOutput(getId(), ExecutableState.READY, 
null, null);
 }
-} else if (result.discarded()) {
-jobService.updateJobOutput(getId(), ExecutableState.DISCARDED, 
null, result.output());
 } else {
 setEndTime(System.currentTimeMillis());
 jobService.updateJobOutput(getId(), ExecutableState.ERROR, null, 
result.output());

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
index 2347e7d..760a574 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
@@ -49,10 +49,6 @@ public final class ExecuteResult {
 return state == State.SUCCEED;
 }
 
-public boolean discarded() {
-return state == State.DISCARDED;
-}
-
 public String output() {
 return output;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/bc2ec1bf/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
--
diff --git 
a/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java 
b/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
deleted file mode 100644
index 9362e18..000
--- a/core-job/src/test/java/org/apache/kylin/job/DiscardedTestExecutable.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more

[3/6] kylin git commit: Revert "KYLIN-2147 Move the creation of HTable after cube be built"

2016-11-03 Thread shaofengshi

Revert "KYLIN-2147 Move the creation of HTable after cube be built"

This reverts commit b6f608f7e145b32f2c9c50bb2b719b40c1c2eb8b.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/df9a8086
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/df9a8086
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/df9a8086

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: df9a80869701f93937b074bae7d73167420208bd
Parents: 669482d
Author: shaofengshi 
Authored: Thu Nov 3 13:27:44 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 3 13:27:44 2016 +0800

--
 .../apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/df9a8086/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index a063efe..c4df354 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -49,11 +49,11 @@ public class HBaseMROutput2Transition implements IMROutput2 
{
 
 @Override
 public void addStepPhase2_BuildDictionary(DefaultChainedExecutable 
jobFlow) {
+
jobFlow.addTask(steps.createCreateHTableStepWithStats(jobFlow.getId()));
 }
 
 @Override
 public void addStepPhase3_BuildCube(DefaultChainedExecutable 
jobFlow) {
-
jobFlow.addTask(steps.createCreateHTableStepWithStats(jobFlow.getId()));
 
jobFlow.addTask(steps.createConvertCuboidToHfileStep(jobFlow.getId()));
 jobFlow.addTask(steps.createBulkLoadStep(jobFlow.getId()));
 }

[6/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread shaofengshi

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8b748cae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8b748cae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8b748cae

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 8b748cae340b78095f216c44ce318c9612b2330c
Parents: 9a2fed5
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi 
Committed: Fri Nov 4 09:47:53 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java|  2 +-
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 48 files changed, 392 insertions(+), 448 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8b748cae/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.

[4/6] kylin git commit: KYLIN-2154 source table rows will be skipped if TOPN's group column contains NULL values

2016-11-03 Thread shaofengshi

KYLIN-2154 source table rows will be skipped if TOPN's group column contains 
NULL values


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a2fed5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a2fed5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a2fed5b

Branch: refs/heads/v1.6.0-rc1-hbase1.x
Commit: 9a2fed5bad2747f4570f92fd4f3f5380ca2c8ff4
Parents: df9a808
Author: Hongbin Ma 
Authored: Wed Nov 2 22:28:35 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 3 13:41:44 2016 +0800

--
 .../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 9 +++--
 .../apache/kylin/storage/gtrecord/CubeSegmentScanner.java   | 5 -
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9a2fed5b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 7e788e4..f34c57c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.measure.topn;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -142,8 +143,12 @@ public class TopNMeasureType extends 
MeasureType {
 final ByteArray key = new ByteArray(keyLength);
 int offset = 0;
 for (int i = 0; i < dimensionEncodings.length; i++) {
-byte[] valueBytes = Bytes.toBytes(values[i + 1]);
-dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+if (values[i + 1] == null) {
+Arrays.fill(key.array(), offset, offset + 
dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL);
+} else {
+byte[] valueBytes = Bytes.toBytes(values[i + 1]);
+dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+}
 offset += dimensionEncodings[i].getLengthOfEncoding();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9a2fed5b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 6e19c09..c6a6daa 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -53,6 +53,9 @@ public class CubeSegmentScanner implements IGTScanner {
 
 public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, 
Set dimensions, Set groups, //
 Collection metrics, TupleFilter originalfilter, 
StorageContext context, String gtStorage) {
+
+logger.info("Init CubeSegmentScanner for segment {}", 
cubeSeg.getName());
+
 this.cuboid = cuboid;
 this.cubeSeg = cubeSeg;
 
@@ -61,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner {
 //is working on its own copy
 byte[] serialize = TupleFilterSerializer.serialize(originalfilter, 
StringCodeSystem.INSTANCE);
 TupleFilter filter = TupleFilterSerializer.deserialize(serialize, 
StringCodeSystem.INSTANCE);
-
+
 // translate FunctionTupleFilter to IN clause
 ITupleFilterTransformer translator = new 
BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
 filter = translator.transform(filter);

kylin git commit: bug fix

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21 44d4fe8c5 -> 52076862c


bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/52076862
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/52076862
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/52076862

Branch: refs/heads/yang21
Commit: 52076862c098f802d6485c893663f5815f4b46f7
Parents: 44d4fe8
Author: Hongbin Ma 
Authored: Fri Nov 4 09:36:12 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 4 09:36:12 2016 +0800

--
 .../org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/52076862/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index ff729f4..3c73dc6 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -98,6 +98,10 @@ public class CubeStorageQuery implements IStorageQuery {
 
 @Override
 public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, 
TupleInfo returnTupleInfo) {
+
+//deal with participant columns in subquery join
+sqlDigest.includeSubqueryJoinParticipants();
+
 //cope with queries with no aggregations
 RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);

[3/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/822be470
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/822be470
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/822be470

Branch: refs/heads/yang21-cdh5.7
Commit: 822be47080c861354cb290be6ae8218128a41a7a
Parents: 44d4fe8
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 4 09:09:35 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/822be470/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[2/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/822be470/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/822be470/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[1/4] kylin git commit: bug fix: if outer query does not have aggr, join columns with subquery should not be added to groupbycolumns [Forced Update!]

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 7398d08b8 -> f03f26af9 (forced update)


bug fix: if outer query does not have aggr, join columns with subquery should 
not be added to groupbycolumns


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/44d4fe8c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/44d4fe8c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/44d4fe8c

Branch: refs/heads/yang21-cdh5.7
Commit: 44d4fe8c590dd2f7822acce40715832cc7a08432
Parents: cfd78bb
Author: Hongbin Ma 
Authored: Fri Nov 4 08:59:12 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 4 08:59:12 2016 +0800

--
 .../org/apache/kylin/metadata/realization/SQLDigest.java| 9 +
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java  | 7 +--
 2 files changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/44d4fe8c/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 3aa39e4..e60d785 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -70,6 +70,15 @@ public class SQLDigest {
 //the reason to choose aggregations rather than metricColumns is 
because the former is set earlier at implOLAP
 }
 
+public void includeSubqueryJoinParticipants() {
+if (this.isRawQuery) {
+this.allColumns.addAll(this.subqueryJoinParticipants);
+} else {
+this.groupbyColumns.addAll(this.subqueryJoinParticipants);
+this.allColumns.addAll(this.subqueryJoinParticipants);
+}
+}
+
 @Override
 public String toString() {
 return "fact table " + this.factTable + "," + //

http://git-wip-us.apache.org/repos/asf/kylin/blob/44d4fe8c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 86d66af..7dc6afa 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -75,12 +75,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, 
TupleInfo returnTupleInfo) {
 
 //deal with participant columns in subquery join
-if (sqlDigest.isRawQuery) {
-sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
-} else {
-
sqlDigest.groupbyColumns.addAll(sqlDigest.subqueryJoinParticipants);
-sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
-}
+sqlDigest.includeSubqueryJoinParticipants();
 
 //cope with queries with no aggregations
 RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);

[1/3] kylin git commit: bug fix: if outer query does not have aggr, join columns with subquery should not be added to groupbycolumns [Forced Update!]

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x b0bb09d17 -> 822be4708 (forced update)


bug fix: if outer query does not have aggr, join columns with subquery should 
not be added to groupbycolumns


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/44d4fe8c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/44d4fe8c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/44d4fe8c

Branch: refs/heads/yang21-hbase1.x
Commit: 44d4fe8c590dd2f7822acce40715832cc7a08432
Parents: cfd78bb
Author: Hongbin Ma 
Authored: Fri Nov 4 08:59:12 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 4 08:59:12 2016 +0800

--
 .../org/apache/kylin/metadata/realization/SQLDigest.java| 9 +
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java  | 7 +--
 2 files changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/44d4fe8c/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 3aa39e4..e60d785 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -70,6 +70,15 @@ public class SQLDigest {
 //the reason to choose aggregations rather than metricColumns is 
because the former is set earlier at implOLAP
 }
 
+public void includeSubqueryJoinParticipants() {
+if (this.isRawQuery) {
+this.allColumns.addAll(this.subqueryJoinParticipants);
+} else {
+this.groupbyColumns.addAll(this.subqueryJoinParticipants);
+this.allColumns.addAll(this.subqueryJoinParticipants);
+}
+}
+
 @Override
 public String toString() {
 return "fact table " + this.factTable + "," + //

http://git-wip-us.apache.org/repos/asf/kylin/blob/44d4fe8c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 86d66af..7dc6afa 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -75,12 +75,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, 
TupleInfo returnTupleInfo) {
 
 //deal with participant columns in subquery join
-if (sqlDigest.isRawQuery) {
-sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
-} else {
-
sqlDigest.groupbyColumns.addAll(sqlDigest.subqueryJoinParticipants);
-sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
-}
+sqlDigest.includeSubqueryJoinParticipants();
 
 //cope with queries with no aggregations
 RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);

[3/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/822be470
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/822be470
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/822be470

Branch: refs/heads/yang21-hbase1.x
Commit: 822be47080c861354cb290be6ae8218128a41a7a
Parents: 44d4fe8
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 4 09:09:35 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/822be470/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[2/3] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/822be470/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/822be470/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

kylin git commit: bug fix: if outer query does not have aggr, join columns with subquery should not be added to groupbycolumns

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21 cfd78bba0 -> 44d4fe8c5


bug fix: if outer query does not have aggr, join columns with subquery should 
not be added to groupbycolumns


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/44d4fe8c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/44d4fe8c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/44d4fe8c

Branch: refs/heads/yang21
Commit: 44d4fe8c590dd2f7822acce40715832cc7a08432
Parents: cfd78bb
Author: Hongbin Ma 
Authored: Fri Nov 4 08:59:12 2016 +0800
Committer: Hongbin Ma 
Committed: Fri Nov 4 08:59:12 2016 +0800

--
 .../org/apache/kylin/metadata/realization/SQLDigest.java| 9 +
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java  | 7 +--
 2 files changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/44d4fe8c/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 3aa39e4..e60d785 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -70,6 +70,15 @@ public class SQLDigest {
 //the reason to choose aggregations rather than metricColumns is 
because the former is set earlier at implOLAP
 }
 
+public void includeSubqueryJoinParticipants() {
+if (this.isRawQuery) {
+this.allColumns.addAll(this.subqueryJoinParticipants);
+} else {
+this.groupbyColumns.addAll(this.subqueryJoinParticipants);
+this.allColumns.addAll(this.subqueryJoinParticipants);
+}
+}
+
 @Override
 public String toString() {
 return "fact table " + this.factTable + "," + //

http://git-wip-us.apache.org/repos/asf/kylin/blob/44d4fe8c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 86d66af..7dc6afa 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -75,12 +75,7 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, 
TupleInfo returnTupleInfo) {
 
 //deal with participant columns in subquery join
-if (sqlDigest.isRawQuery) {
-sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
-} else {
-
sqlDigest.groupbyColumns.addAll(sqlDigest.subqueryJoinParticipants);
-sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
-}
+sqlDigest.includeSubqueryJoinParticipants();
 
 //cope with queries with no aggregations
 RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);

[2/5] kylin git commit: fix CI

2016-11-03 Thread mahongbin

fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cfd78bba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cfd78bba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cfd78bba

Branch: refs/heads/yang21-cdh5.7
Commit: cfd78bba0eb0eace520e520b1dfd4fc077b9628f
Parents: c55a6fb
Author: Hongbin Ma 
Authored: Thu Nov 3 19:21:21 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 19:21:21 2016 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cfd78bba/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 48b9e77..7503fbf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -90,10 +90,10 @@ public class Cuboid implements Comparable {
 
 public static boolean isValid(CubeDesc cube, long cuboidID) {
 //base cuboid is always valid
-if (cuboidID == getBaseCuboidId(agg.getCubeDesc())) {
+if (cuboidID == getBaseCuboidId(cube)) {
 return true;
 }
-
+
 for (AggregationGroup agg : cube.getAggregationGroups()) {
 if (isValid(agg, cuboidID)) {
 return true;

[3/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0bb09d1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b0bb09d1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[5/5] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-03 Thread mahongbin

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7398d08b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7398d08b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7398d08b

Branch: refs/heads/yang21-cdh5.7
Commit: 7398d08b850ae86d2cf03c30f0390b5ea7c7362f
Parents: b0bb09d
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Thu Nov 3 19:25:57 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7398d08b/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/7398d08b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/7398d08b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7398d08b/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0bb09d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0bb09d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0bb09d1

Branch: refs/heads/yang21-cdh5.7
Commit: b0bb09d17a7cbec462b4b4a60fac813d58761fd8
Parents: cfd78bb
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 19:24:09 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b0bb09d1/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[4/4] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0bb09d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0bb09d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0bb09d1

Branch: refs/heads/yang21-hbase1.x
Commit: b0bb09d17a7cbec462b4b4a60fac813d58761fd8
Parents: cfd78bb
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 19:24:09 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b0bb09d1/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[2/4] kylin git commit: fix CI

2016-11-03 Thread mahongbin

fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cfd78bba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cfd78bba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cfd78bba

Branch: refs/heads/yang21-hbase1.x
Commit: cfd78bba0eb0eace520e520b1dfd4fc077b9628f
Parents: c55a6fb
Author: Hongbin Ma 
Authored: Thu Nov 3 19:21:21 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 19:21:21 2016 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cfd78bba/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 48b9e77..7503fbf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -90,10 +90,10 @@ public class Cuboid implements Comparable {
 
 public static boolean isValid(CubeDesc cube, long cuboidID) {
 //base cuboid is always valid
-if (cuboidID == getBaseCuboidId(agg.getCubeDesc())) {
+if (cuboidID == getBaseCuboidId(cube)) {
 return true;
 }
-
+
 for (AggregationGroup agg : cube.getAggregationGroups()) {
 if (isValid(agg, cuboidID)) {
 return true;

[1/4] kylin git commit: fix CI [Forced Update!]

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 2a8ca1dd6 -> b0bb09d17 (forced update)


fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c55a6fbe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c55a6fbe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c55a6fbe

Branch: refs/heads/yang21-hbase1.x
Commit: c55a6fbeaf2348ffc0e1a0dc6e514812ac0b54e1
Parents: cb0a1a3
Author: Hongbin Ma 
Authored: Thu Nov 3 19:20:09 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 19:20:09 2016 +0800

--
 .../src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java  | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c55a6fbe/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 855aad6..48b9e77 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -89,6 +89,11 @@ public class Cuboid implements Comparable {
 }
 
 public static boolean isValid(CubeDesc cube, long cuboidID) {
+//base cuboid is always valid
+if (cuboidID == getBaseCuboidId(agg.getCubeDesc())) {
+return true;
+}
+
 for (AggregationGroup agg : cube.getAggregationGroups()) {
 if (isValid(agg, cuboidID)) {
 return true;

kylin git commit: fix CI

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21 c55a6fbea -> cfd78bba0


fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cfd78bba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cfd78bba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cfd78bba

Branch: refs/heads/yang21
Commit: cfd78bba0eb0eace520e520b1dfd4fc077b9628f
Parents: c55a6fb
Author: Hongbin Ma 
Authored: Thu Nov 3 19:21:21 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 19:21:21 2016 +0800

--
 core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cfd78bba/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 48b9e77..7503fbf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -90,10 +90,10 @@ public class Cuboid implements Comparable {
 
 public static boolean isValid(CubeDesc cube, long cuboidID) {
 //base cuboid is always valid
-if (cuboidID == getBaseCuboidId(agg.getCubeDesc())) {
+if (cuboidID == getBaseCuboidId(cube)) {
 return true;
 }
-
+
 for (AggregationGroup agg : cube.getAggregationGroups()) {
 if (isValid(agg, cuboidID)) {
 return true;

[1/2] kylin git commit: KYLIN-2135 Enlarge FactDistinctColumns reducer number

2016-11-03 Thread shaofengshi

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2135 [created] 3b01b94c0


KYLIN-2135 Enlarge FactDistinctColumns reducer number

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/04323b02
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/04323b02
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/04323b02

Branch: refs/heads/KYLIN-2135
Commit: 04323b02e40ea1b53e7070ae77ae5025177e5bb4
Parents: 93e3020
Author: kangkaisen 
Authored: Wed Oct 26 19:35:20 2016 +0800
Committer: shaofengshi 
Committed: Thu Nov 3 14:24:59 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 ++
 .../java/org/apache/kylin/cube/CubeManager.java | 35 
 .../kylin/engine/mr/DFSFileTableReader.java | 59 
 .../kylin/engine/mr/common/BatchConstants.java  |  5 ++
 .../mr/steps/FactDistinctColumnPartitioner.java | 11 +---
 .../engine/mr/steps/FactDistinctColumnsJob.java | 18 +-
 .../mr/steps/FactDistinctColumnsMapperBase.java | 17 +-
 .../mr/steps/FactDistinctColumnsReducer.java| 36 ++--
 .../mr/steps/FactDistinctHiveColumnsMapper.java | 12 +++-
 9 files changed, 171 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/04323b02/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index acc4eb1..9ac8142 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -374,6 +374,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.job.allow.empty.segment", "true"));
 }
 
+//UHC: ultra high cardinality columns, contain the ShardByColumns and the 
GlobalDictionaryColumns
+public int getUHCReducerCount() {
+return Integer.parseInt(getOptional("kylin.job.uhc.reducer.count", 
"3"));
+}
+
 public String getOverrideHiveTableLocation(String table) {
 return getOptional("hive.table.location." + table.toUpperCase());
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/04323b02/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 16b468f..c04617d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -31,6 +31,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -43,6 +44,7 @@ import org.apache.kylin.common.persistence.Serializer;
 import org.apache.kylin.common.util.Dictionary;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DictionaryDesc;
 import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
@@ -1047,4 +1049,37 @@ public class CubeManager implements IRealizationProvider 
{
 }
 return holes;
 }
+
+private final String GLOBAL_DICTIONNARY_CLASS = 
"org.apache.kylin.dict.GlobalDictionaryBuilder";
+
+//UHC (ultra high cardinality column): contain the ShardByColumns and the 
GlobalDictionaryColumns
+public int[] getUHCIndex(CubeDesc cubeDesc) throws IOException {
+List factDictCols = getAllDictColumnsOnFact(cubeDesc);
+int[] uhcIndex = new int[factDictCols.size()];
+
+//add GlobalDictionaryColumns
+List dictionaryDescList = cubeDesc.getDictionaries();
+if (dictionaryDescList != null) {
+for (DictionaryDesc dictionaryDesc : dictionaryDescList) {
+if (dictionaryDesc.getBuilderClass() != null && 
dictionaryDesc.getBuilderClass().equalsIgnoreCase(GLOBAL_DICTIONNARY_CLASS)) {
+for (int i = 0; i < factDictCols.size(); i++) {
+if 
(factDictCols.get(i).equals(dictionaryDesc.getColumnRef())) {
+uhcIndex[i] = 1;
+break;
+}
+}
+}
+}
+}
+
+//add ShardByColumns
+Set

[3/9] kylin git commit: KYLIN-1698 minor bug fix

2016-11-03 Thread mahongbin

KYLIN-1698 minor bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/375d4f79
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/375d4f79
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/375d4f79

Branch: refs/heads/yang21-cdh5.7
Commit: 375d4f790ed25d65729cdf2c09e4787cb2c7986f
Parents: 7ef9f2a
Author: Li Yang 
Authored: Thu Nov 3 17:20:42 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 17:20:42 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/util/DateFormat.java   | 4 
 .../main/java/org/apache/kylin/metadata/model/PartitionDesc.java | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/375d4f79/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java 
b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 2472992..1d70a2d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -138,4 +138,8 @@ public class DateFormat {
 }
 return false;
 }
+
+public static boolean isDatePattern(String ptn) {
+return COMPACT_DATE_PATTERN.equals(ptn) || 
DEFAULT_DATE_PATTERN.equals(ptn);
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/375d4f79/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 6487bfa..11b9bdf 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -96,7 +96,7 @@ public class PartitionDesc {
 return false;
 
 DataType type = partitionDateColumnRef.getType();
-return type.isInt();
+return (type.isInt() || type.isBigInt()) && 
DateFormat.isDatePattern(partitionDateFormat);
 }
 
 public boolean partitionColumnIsTimeMillis() {
@@ -104,7 +104,7 @@ public class PartitionDesc {
 return false;
 
 DataType type = partitionDateColumnRef.getType();
-return type.isBigInt();
+return type.isBigInt() && 
!DateFormat.isDatePattern(partitionDateFormat);
 }
 
 public boolean isPartitioned() {

[9/9] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2016-11-03 Thread mahongbin

KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/37529c55
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/37529c55
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/37529c55

Branch: refs/heads/yang21-cdh5.7
Commit: 37529c55212475981bc932a58b310b15537284cb
Parents: 2a8ca1d
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: Hongbin Ma 
Committed: Thu Nov 3 18:39:50 2016 +0800

--
 build/conf/kylin.properties |   3 +
 dev-support/test_all_against_hdp_2_2_4_2_2.sh   |   0
 .../kylin/engine/mr/steps/MockupMapContext.java |  15 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  19 +-
 tool/pom.xml|  12 +
 12 files changed, 431 insertions(+), 1131 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/37529c55/build/conf/kylin.properties
--
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..ebd8cde 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -118,6 +118,9 @@ kylin.job.mapreduce.mapper.input.rows=100
 
 kylin.job.step.timeout=7200
 
+# for secure cdh, filtering hive dependency is risky, so filter nothing
+kylin.job.dependency.filterlist=[^,]+
+
 ### CUBE ###
 
 # 'auto', 'inmem', 'layer' or 'random' for testing

http://git-wip-us.apache.org/repos/asf/kylin/blob/37529c55/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh 
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/kylin/blob/37529c55/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
 outKV[0] = key;
 outKV[1] = value;
 }
+
 }
 
 @Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
 throw new NotImplementedException();
 }
 
+
 @Override
 public float getProgress() {
 throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
 }
 
 @Override
-public RawComparator getSortComparator() {
+public boolean userClassesTakesPrecedence() {
 throw new NotImplementedException();
 }
 
 @Override
-public String getJar() {
+public RawComparator getSortComparator() {
 throw new NotImplementedException();
 }
 
 @Override
-public RawComparator getGroupingComparator() {
+public String getJar() {
 throw new NotImplementedException();
 }
 
@@ -221,7 +223,7 @@ public class MockupMapContext {
 
 @Override
 public boolean getProfileEnabled() {
-throw new NotImplementedException();
+return false;
 }
 
 @Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
+
+@Override
+public RawComparator getGroupingComparator() {
+return null;
+}
 });
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/37529c55/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml

[4/9] kylin git commit: minor, clearer log on 'Value not exist!' error

2016-11-03 Thread mahongbin

minor, clearer log on 'Value not exist!' error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca2e0717
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca2e0717
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca2e0717

Branch: refs/heads/yang21-cdh5.7
Commit: ca2e071706464e76e503ebdb072b53466121ee71
Parents: 375d4f7
Author: Li Yang 
Authored: Thu Nov 3 18:21:55 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 18:21:55 2016 +0800

--
 .../main/java/org/apache/kylin/common/util/Dictionary.java   | 2 +-
 .../java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java | 8 +++-
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca2e0717/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java 
b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 86ad5ff..0fb299c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -159,7 +159,7 @@ abstract public class Dictionary implements Serializable 
{
 else {
 int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
 if (id < 0)
-throw new IllegalArgumentException("Value not exists!");
+throw new IllegalArgumentException("Value '" + 
Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, 
offset, len) + ") not exists!");
 return id;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca2e0717/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index f95cc21..aeeb893 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -118,7 +118,13 @@ public class CubeCodeSystem implements IGTCodeSystem {
 if (dictEnc.getRoundingFlag() != roundingFlag) {
 serializer = dictEnc.copy(roundingFlag).asDataTypeSerializer();
 }
-serializer.serialize(value, buf);
+try {
+serializer.serialize(value, buf);
+} catch (IllegalArgumentException ex) {
+IllegalArgumentException rewordEx = new 
IllegalArgumentException("Column " + col + " value '" + value + "' met 
dictionary error: " + ex.getMessage());
+rewordEx.setStackTrace(ex.getStackTrace());
+throw rewordEx;
+}
 } else {
 if (value instanceof String) {
 // for dimensions; measures are converted by MeasureIngestor 
before reaching this point

[1/9] kylin git commit: KYLIN-1945 Cuboid.translateToValidCuboid method throw exception while cube building or query execute [Forced Update!]

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21-cdh5.7 5ca33805b -> 37529c552 (forced update)


KYLIN-1945 Cuboid.translateToValidCuboid method throw exception while cube 
building or query execute


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/631e6f1c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/631e6f1c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/631e6f1c

Branch: refs/heads/yang21-cdh5.7
Commit: 631e6f1c1a24862b588b8504bc69410830e4cd79
Parents: 6be0888
Author: Hongbin Ma 
Authored: Thu Nov 3 16:01:04 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 16:01:04 2016 +0800

--
 .../main/java/org/apache/kylin/cube/cuboid/Cuboid.java| 10 +-
 .../org/apache/kylin/cube/model/AggregationGroup.java |  4 
 2 files changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/631e6f1c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index e4e718d..855aad6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -89,16 +89,11 @@ public class Cuboid implements Comparable {
 }
 
 public static boolean isValid(CubeDesc cube, long cuboidID) {
-if (cuboidID == getBaseCuboidId(cube)) {
-return true;
-}
-
 for (AggregationGroup agg : cube.getAggregationGroups()) {
 if (isValid(agg, cuboidID)) {
 return true;
 }
 }
-
 return false;
 }
 
@@ -231,6 +226,11 @@ public class Cuboid implements Comparable {
 if ((cuboidID & mandatoryColumnMask) != mandatoryColumnMask) {
 return false;
 } else {
+//base cuboid is always valid
+if (cuboidID == getBaseCuboidId(agg.getCubeDesc())) {
+return true;
+}
+
 //cuboid with only mandatory columns maybe valid
 return agg.isMandatoryOnlyValid() || (cuboidID & 
~mandatoryColumnMask) != 0;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/631e6f1c/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 601ee0a..5f780d0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -281,4 +281,8 @@ public class AggregationGroup {
 public boolean isMandatoryOnlyValid() {
 return isMandatoryOnlyValid;
 }
+
+public CubeDesc getCubeDesc() {
+return cubeDesc;
+}
 }

[5/9] kylin git commit: KYLIN-2156: support filters like a != 0

2016-11-03 Thread mahongbin

KYLIN-2156: support filters like a != 0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc01e155
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc01e155
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc01e155

Branch: refs/heads/yang21-cdh5.7
Commit: dc01e15508c6ffba16599342385f1622c3494787
Parents: ca2e071
Author: Hongbin Ma 
Authored: Thu Nov 3 16:49:42 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 18:32:59 2016 +0800

--
 .../main/java/org/apache/kylin/rest/util/QueryUtil.java   | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dc01e155/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index eb73e8f..59a4a78 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -95,6 +95,7 @@ public class QueryUtil {
 private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + 
"GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = 
Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + 
"1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!="+ 
S0, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_INTERVAL = Pattern.compile("interval" 
+ SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_CONCAT = 
Pattern.compile("concat\\(.+?\\)");//non-greedy
 private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);
@@ -129,6 +130,15 @@ public class QueryUtil {
 sql = sql.substring(0, m.start()) + " COUNT(1) " + 
sql.substring(m.end());
 }
 
+// Case: !=
+// Replace it with <>
+while (true) {
+m = PTN_NOT_EQ.matcher(sql);
+if (!m.find())
+break;
+sql = sql.substring(0, m.start()) + " <> " + 
sql.substring(m.end());
+}
+
 // ( date '2001-09-28' + interval floor(1) day ) generated by 
cognos
 // calcite only recognizes date '2001-09-28' + interval '1' day
 while (true) {

[2/9] kylin git commit: minor, update default engines in cloned cube

2016-11-03 Thread mahongbin

minor, update default engines in cloned cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7ef9f2ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7ef9f2ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7ef9f2ae

Branch: refs/heads/yang21-cdh5.7
Commit: 7ef9f2ae6f1ba0b3fc870dcecc78745461be198f
Parents: 631e6f1
Author: Li Yang 
Authored: Thu Nov 3 16:39:27 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 16:39:39 2016 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java   | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7ef9f2ae/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8dfaed6..64fde81 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -28,6 +28,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
@@ -359,7 +360,11 @@ public class CubeController extends BasicController {
 
 CubeDesc cubeDesc = cube.getDescriptor();
 CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
+
+KylinConfig config = cubeService.getConfig();
 newCubeDesc.setName(newCubeName);
+newCubeDesc.setEngineType(config.getDefaultCubeEngine());
+newCubeDesc.setStorageType(config.getDefaultStorageEngine());
 
 CubeInstance newCube;
 try {

[5/8] kylin git commit: KYLIN-2156: support filters like a != 0

2016-11-03 Thread mahongbin

KYLIN-2156: support filters like a != 0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc01e155
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc01e155
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc01e155

Branch: refs/heads/yang21-hbase1.x
Commit: dc01e15508c6ffba16599342385f1622c3494787
Parents: ca2e071
Author: Hongbin Ma 
Authored: Thu Nov 3 16:49:42 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 18:32:59 2016 +0800

--
 .../main/java/org/apache/kylin/rest/util/QueryUtil.java   | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dc01e155/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index eb73e8f..59a4a78 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -95,6 +95,7 @@ public class QueryUtil {
 private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + 
"GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = 
Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + 
"1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!="+ 
S0, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_INTERVAL = Pattern.compile("interval" 
+ SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_CONCAT = 
Pattern.compile("concat\\(.+?\\)");//non-greedy
 private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);
@@ -129,6 +130,15 @@ public class QueryUtil {
 sql = sql.substring(0, m.start()) + " COUNT(1) " + 
sql.substring(m.end());
 }
 
+// Case: !=
+// Replace it with <>
+while (true) {
+m = PTN_NOT_EQ.matcher(sql);
+if (!m.find())
+break;
+sql = sql.substring(0, m.start()) + " <> " + 
sql.substring(m.end());
+}
+
 // ( date '2001-09-28' + interval floor(1) day ) generated by 
cognos
 // calcite only recognizes date '2001-09-28' + interval '1' day
 while (true) {

[8/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2a8ca1dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2a8ca1dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2a8ca1dd

Branch: refs/heads/yang21-hbase1.x
Commit: 2a8ca1dd641a88a0ac1c0b22cef4047666ee0c90
Parents: cb0a1a3
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 18:38:03 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2a8ca1dd/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[4/8] kylin git commit: minor, clearer log on 'Value not exist!' error

2016-11-03 Thread mahongbin

minor, clearer log on 'Value not exist!' error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca2e0717
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca2e0717
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca2e0717

Branch: refs/heads/yang21-hbase1.x
Commit: ca2e071706464e76e503ebdb072b53466121ee71
Parents: 375d4f7
Author: Li Yang 
Authored: Thu Nov 3 18:21:55 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 18:21:55 2016 +0800

--
 .../main/java/org/apache/kylin/common/util/Dictionary.java   | 2 +-
 .../java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java | 8 +++-
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca2e0717/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java 
b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 86ad5ff..0fb299c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -159,7 +159,7 @@ abstract public class Dictionary implements Serializable 
{
 else {
 int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
 if (id < 0)
-throw new IllegalArgumentException("Value not exists!");
+throw new IllegalArgumentException("Value '" + 
Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, 
offset, len) + ") not exists!");
 return id;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca2e0717/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index f95cc21..aeeb893 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -118,7 +118,13 @@ public class CubeCodeSystem implements IGTCodeSystem {
 if (dictEnc.getRoundingFlag() != roundingFlag) {
 serializer = dictEnc.copy(roundingFlag).asDataTypeSerializer();
 }
-serializer.serialize(value, buf);
+try {
+serializer.serialize(value, buf);
+} catch (IllegalArgumentException ex) {
+IllegalArgumentException rewordEx = new 
IllegalArgumentException("Column " + col + " value '" + value + "' met 
dictionary error: " + ex.getMessage());
+rewordEx.setStackTrace(ex.getStackTrace());
+throw rewordEx;
+}
 } else {
 if (value instanceof String) {
 // for dimensions; measures are converted by MeasureIngestor 
before reaching this point

[2/8] kylin git commit: minor, update default engines in cloned cube

2016-11-03 Thread mahongbin

minor, update default engines in cloned cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7ef9f2ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7ef9f2ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7ef9f2ae

Branch: refs/heads/yang21-hbase1.x
Commit: 7ef9f2ae6f1ba0b3fc870dcecc78745461be198f
Parents: 631e6f1
Author: Li Yang 
Authored: Thu Nov 3 16:39:27 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 16:39:39 2016 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java   | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7ef9f2ae/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8dfaed6..64fde81 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -28,6 +28,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
@@ -359,7 +360,11 @@ public class CubeController extends BasicController {
 
 CubeDesc cubeDesc = cube.getDescriptor();
 CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
+
+KylinConfig config = cubeService.getConfig();
 newCubeDesc.setName(newCubeName);
+newCubeDesc.setEngineType(config.getDefaultCubeEngine());
+newCubeDesc.setStorageType(config.getDefaultStorageEngine());
 
 CubeInstance newCube;
 try {

[6/8] kylin git commit: if outer query does not have aggr, join columns with subquery should not be added to groupbycolumns

2016-11-03 Thread mahongbin

if outer query does not have aggr, join columns with subquery should not be 
added to groupbycolumns


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cb0a1a39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cb0a1a39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cb0a1a39

Branch: refs/heads/yang21-hbase1.x
Commit: cb0a1a39555994b481da7b9151b4bc8a3955c179
Parents: dc01e15
Author: Hongbin Ma 
Authored: Thu Nov 3 18:32:19 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 18:32:59 2016 +0800

--
 .../org/apache/kylin/metadata/realization/SQLDigest.java | 4 +++-
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 8 
 .../java/org/apache/kylin/storage/hbase/ITStorageTest.java   | 2 +-
 .../java/org/apache/kylin/query/relnode/OLAPContext.java | 3 ++-
 .../java/org/apache/kylin/query/relnode/OLAPJoinRel.java | 5 ++---
 5 files changed, 16 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cb0a1a39/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index d2bba66..3aa39e4 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -39,6 +39,7 @@ public class SQLDigest {
 public Collection joinDescs;
 public Collection allColumns;
 public Collection groupbyColumns;
+public Collection subqueryJoinParticipants;
 public Collection filterColumns;
 public Collection metricColumns;
 public Collection aggregations;
@@ -48,12 +49,13 @@ public class SQLDigest {
 
 //initialized when 
org.apache.kylin.query.routing.QueryRouter.selectRealization()
 public SQLDigest(String factTable, TupleFilter filter, 
Collection joinDescs, Collection allColumns, //
-Collection groupbyColumns, Collection 
filterColumns, Collection aggregatedColumns, 
Collection aggregateFunnc, Collection sortMeasures, 
Collection sortOrders) {
+Collection groupbyColumns, Collection 
subqueryJoinParticipants, Collection filterColumns, 
Collection aggregatedColumns, Collection 
aggregateFunnc, Collection sortMeasures, Collection 
sortOrders) {
 this.factTable = factTable;
 this.filter = filter;
 this.joinDescs = joinDescs;
 this.allColumns = allColumns;
 this.groupbyColumns = groupbyColumns;
+this.subqueryJoinParticipants = subqueryJoinParticipants;
 this.filterColumns = filterColumns;
 this.metricColumns = aggregatedColumns;
 this.aggregations = aggregateFunnc;

http://git-wip-us.apache.org/repos/asf/kylin/blob/cb0a1a39/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index fa23be1..86d66af 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -74,6 +74,14 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 @Override
 public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, 
TupleInfo returnTupleInfo) {
 
+//deal with participant columns in subquery join
+if (sqlDigest.isRawQuery) {
+sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
+} else {
+
sqlDigest.groupbyColumns.addAll(sqlDigest.subqueryJoinParticipants);
+sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
+}
+
 //cope with queries with no aggregations
 RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/cb0a1a39/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
index 136342d..bb3257b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
+++

[3/8] kylin git commit: KYLIN-1698 minor bug fix

2016-11-03 Thread mahongbin

KYLIN-1698 minor bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/375d4f79
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/375d4f79
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/375d4f79

Branch: refs/heads/yang21-hbase1.x
Commit: 375d4f790ed25d65729cdf2c09e4787cb2c7986f
Parents: 7ef9f2a
Author: Li Yang 
Authored: Thu Nov 3 17:20:42 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 17:20:42 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/util/DateFormat.java   | 4 
 .../main/java/org/apache/kylin/metadata/model/PartitionDesc.java | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/375d4f79/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java 
b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 2472992..1d70a2d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -138,4 +138,8 @@ public class DateFormat {
 }
 return false;
 }
+
+public static boolean isDatePattern(String ptn) {
+return COMPACT_DATE_PATTERN.equals(ptn) || 
DEFAULT_DATE_PATTERN.equals(ptn);
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/375d4f79/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 6487bfa..11b9bdf 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -96,7 +96,7 @@ public class PartitionDesc {
 return false;
 
 DataType type = partitionDateColumnRef.getType();
-return type.isInt();
+return (type.isInt() || type.isBigInt()) && 
DateFormat.isDatePattern(partitionDateFormat);
 }
 
 public boolean partitionColumnIsTimeMillis() {
@@ -104,7 +104,7 @@ public class PartitionDesc {
 return false;
 
 DataType type = partitionDateColumnRef.getType();
-return type.isBigInt();
+return type.isBigInt() && 
!DateFormat.isDatePattern(partitionDateFormat);
 }
 
 public boolean isPartitioned() {

[7/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/2a8ca1dd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2a8ca1dd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[1/8] kylin git commit: KYLIN-1945 Cuboid.translateToValidCuboid method throw exception while cube building or query execute [Forced Update!]

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x a93c2758a -> 2a8ca1dd6 (forced update)


KYLIN-1945 Cuboid.translateToValidCuboid method throw exception while cube 
building or query execute


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/631e6f1c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/631e6f1c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/631e6f1c

Branch: refs/heads/yang21-hbase1.x
Commit: 631e6f1c1a24862b588b8504bc69410830e4cd79
Parents: 6be0888
Author: Hongbin Ma 
Authored: Thu Nov 3 16:01:04 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 16:01:04 2016 +0800

--
 .../main/java/org/apache/kylin/cube/cuboid/Cuboid.java| 10 +-
 .../org/apache/kylin/cube/model/AggregationGroup.java |  4 
 2 files changed, 9 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/631e6f1c/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index e4e718d..855aad6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -89,16 +89,11 @@ public class Cuboid implements Comparable {
 }
 
 public static boolean isValid(CubeDesc cube, long cuboidID) {
-if (cuboidID == getBaseCuboidId(cube)) {
-return true;
-}
-
 for (AggregationGroup agg : cube.getAggregationGroups()) {
 if (isValid(agg, cuboidID)) {
 return true;
 }
 }
-
 return false;
 }
 
@@ -231,6 +226,11 @@ public class Cuboid implements Comparable {
 if ((cuboidID & mandatoryColumnMask) != mandatoryColumnMask) {
 return false;
 } else {
+//base cuboid is always valid
+if (cuboidID == getBaseCuboidId(agg.getCubeDesc())) {
+return true;
+}
+
 //cuboid with only mandatory columns maybe valid
 return agg.isMandatoryOnlyValid() || (cuboidID & 
~mandatoryColumnMask) != 0;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/631e6f1c/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 601ee0a..5f780d0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -281,4 +281,8 @@ public class AggregationGroup {
 public boolean isMandatoryOnlyValid() {
 return isMandatoryOnlyValid;
 }
+
+public CubeDesc getCubeDesc() {
+return cubeDesc;
+}
 }

[1/2] kylin git commit: if outer query does not have aggr, join columns with subquery should not be added to groupbycolumns

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21 ca2e07170 -> cb0a1a395


if outer query does not have aggr, join columns with subquery should not be 
added to groupbycolumns


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cb0a1a39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cb0a1a39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cb0a1a39

Branch: refs/heads/yang21
Commit: cb0a1a39555994b481da7b9151b4bc8a3955c179
Parents: dc01e15
Author: Hongbin Ma 
Authored: Thu Nov 3 18:32:19 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 18:32:59 2016 +0800

--
 .../org/apache/kylin/metadata/realization/SQLDigest.java | 4 +++-
 .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java   | 8 
 .../java/org/apache/kylin/storage/hbase/ITStorageTest.java   | 2 +-
 .../java/org/apache/kylin/query/relnode/OLAPContext.java | 3 ++-
 .../java/org/apache/kylin/query/relnode/OLAPJoinRel.java | 5 ++---
 5 files changed, 16 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cb0a1a39/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index d2bba66..3aa39e4 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -39,6 +39,7 @@ public class SQLDigest {
 public Collection joinDescs;
 public Collection allColumns;
 public Collection groupbyColumns;
+public Collection subqueryJoinParticipants;
 public Collection filterColumns;
 public Collection metricColumns;
 public Collection aggregations;
@@ -48,12 +49,13 @@ public class SQLDigest {
 
 //initialized when 
org.apache.kylin.query.routing.QueryRouter.selectRealization()
 public SQLDigest(String factTable, TupleFilter filter, 
Collection joinDescs, Collection allColumns, //
-Collection groupbyColumns, Collection 
filterColumns, Collection aggregatedColumns, 
Collection aggregateFunnc, Collection sortMeasures, 
Collection sortOrders) {
+Collection groupbyColumns, Collection 
subqueryJoinParticipants, Collection filterColumns, 
Collection aggregatedColumns, Collection 
aggregateFunnc, Collection sortMeasures, Collection 
sortOrders) {
 this.factTable = factTable;
 this.filter = filter;
 this.joinDescs = joinDescs;
 this.allColumns = allColumns;
 this.groupbyColumns = groupbyColumns;
+this.subqueryJoinParticipants = subqueryJoinParticipants;
 this.filterColumns = filterColumns;
 this.metricColumns = aggregatedColumns;
 this.aggregations = aggregateFunnc;

http://git-wip-us.apache.org/repos/asf/kylin/blob/cb0a1a39/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index fa23be1..86d66af 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -74,6 +74,14 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 @Override
 public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, 
TupleInfo returnTupleInfo) {
 
+//deal with participant columns in subquery join
+if (sqlDigest.isRawQuery) {
+sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
+} else {
+
sqlDigest.groupbyColumns.addAll(sqlDigest.subqueryJoinParticipants);
+sqlDigest.allColumns.addAll(sqlDigest.subqueryJoinParticipants);
+}
+
 //cope with queries with no aggregations
 RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/cb0a1a39/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java 
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
index 136342d..bb3257b 100644
---

[2/2] kylin git commit: KYLIN-2156: support filters like a != 0

2016-11-03 Thread mahongbin

KYLIN-2156: support filters like a != 0


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dc01e155
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dc01e155
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dc01e155

Branch: refs/heads/yang21
Commit: dc01e15508c6ffba16599342385f1622c3494787
Parents: ca2e071
Author: Hongbin Ma 
Authored: Thu Nov 3 16:49:42 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 18:32:59 2016 +0800

--
 .../main/java/org/apache/kylin/rest/util/QueryUtil.java   | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dc01e155/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java 
b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index eb73e8f..59a4a78 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -95,6 +95,7 @@ public class QueryUtil {
 private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + 
"GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = 
Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + 
"1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + 
S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
+private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!="+ 
S0, Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_INTERVAL = Pattern.compile("interval" 
+ SM + "(floor\\()([\\d\\.]+)(\\))" + SM + 
"(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
 private static final Pattern PTN_CONCAT = 
Pattern.compile("concat\\(.+?\\)");//non-greedy
 private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = 
Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);
@@ -129,6 +130,15 @@ public class QueryUtil {
 sql = sql.substring(0, m.start()) + " COUNT(1) " + 
sql.substring(m.end());
 }
 
+// Case: !=
+// Replace it with <>
+while (true) {
+m = PTN_NOT_EQ.matcher(sql);
+if (!m.find())
+break;
+sql = sql.substring(0, m.start()) + " <> " + 
sql.substring(m.end());
+}
+
 // ( date '2001-09-28' + interval floor(1) day ) generated by 
cognos
 // calcite only recognizes date '2001-09-28' + interval '1' day
 while (true) {

kylin git commit: minor, clearer log on 'Value not exist!' error

2016-11-03 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 139595538 -> 235b123d0


minor, clearer log on 'Value not exist!' error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/235b123d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/235b123d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/235b123d

Branch: refs/heads/master
Commit: 235b123d09c629b685cbaf029f205bbeab887d6f
Parents: 1395955
Author: Li Yang 
Authored: Thu Nov 3 18:21:55 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 18:23:01 2016 +0800

--
 .../main/java/org/apache/kylin/common/util/Dictionary.java   | 2 +-
 .../java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java | 8 +++-
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/235b123d/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java 
b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 86ad5ff..0fb299c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -159,7 +159,7 @@ abstract public class Dictionary implements Serializable 
{
 else {
 int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
 if (id < 0)
-throw new IllegalArgumentException("Value not exists!");
+throw new IllegalArgumentException("Value '" + 
Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, 
offset, len) + ") not exists!");
 return id;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/235b123d/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index f95cc21..aeeb893 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -118,7 +118,13 @@ public class CubeCodeSystem implements IGTCodeSystem {
 if (dictEnc.getRoundingFlag() != roundingFlag) {
 serializer = dictEnc.copy(roundingFlag).asDataTypeSerializer();
 }
-serializer.serialize(value, buf);
+try {
+serializer.serialize(value, buf);
+} catch (IllegalArgumentException ex) {
+IllegalArgumentException rewordEx = new 
IllegalArgumentException("Column " + col + " value '" + value + "' met 
dictionary error: " + ex.getMessage());
+rewordEx.setStackTrace(ex.getStackTrace());
+throw rewordEx;
+}
 } else {
 if (value instanceof String) {
 // for dimensions; measures are converted by MeasureIngestor 
before reaching this point

kylin git commit: minor, clearer log on 'Value not exist!' error

2016-11-03 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang21 375d4f790 -> ca2e07170


minor, clearer log on 'Value not exist!' error


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca2e0717
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca2e0717
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca2e0717

Branch: refs/heads/yang21
Commit: ca2e071706464e76e503ebdb072b53466121ee71
Parents: 375d4f7
Author: Li Yang 
Authored: Thu Nov 3 18:21:55 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 18:21:55 2016 +0800

--
 .../main/java/org/apache/kylin/common/util/Dictionary.java   | 2 +-
 .../java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java | 8 +++-
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ca2e0717/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java 
b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 86ad5ff..0fb299c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -159,7 +159,7 @@ abstract public class Dictionary implements Serializable 
{
 else {
 int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
 if (id < 0)
-throw new IllegalArgumentException("Value not exists!");
+throw new IllegalArgumentException("Value '" + 
Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, 
offset, len) + ") not exists!");
 return id;
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ca2e0717/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index f95cc21..aeeb893 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -118,7 +118,13 @@ public class CubeCodeSystem implements IGTCodeSystem {
 if (dictEnc.getRoundingFlag() != roundingFlag) {
 serializer = dictEnc.copy(roundingFlag).asDataTypeSerializer();
 }
-serializer.serialize(value, buf);
+try {
+serializer.serialize(value, buf);
+} catch (IllegalArgumentException ex) {
+IllegalArgumentException rewordEx = new 
IllegalArgumentException("Column " + col + " value '" + value + "' met 
dictionary error: " + ex.getMessage());
+rewordEx.setStackTrace(ex.getStackTrace());
+throw rewordEx;
+}
 } else {
 if (value instanceof String) {
 // for dimensions; measures are converted by MeasureIngestor 
before reaching this point

kylin git commit: KYLIN-1698 minor bug fix

2016-11-03 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang21 7ef9f2ae6 -> 375d4f790


KYLIN-1698 minor bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/375d4f79
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/375d4f79
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/375d4f79

Branch: refs/heads/yang21
Commit: 375d4f790ed25d65729cdf2c09e4787cb2c7986f
Parents: 7ef9f2a
Author: Li Yang 
Authored: Thu Nov 3 17:20:42 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 17:20:42 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/util/DateFormat.java   | 4 
 .../main/java/org/apache/kylin/metadata/model/PartitionDesc.java | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/375d4f79/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java 
b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 2472992..1d70a2d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -138,4 +138,8 @@ public class DateFormat {
 }
 return false;
 }
+
+public static boolean isDatePattern(String ptn) {
+return COMPACT_DATE_PATTERN.equals(ptn) || 
DEFAULT_DATE_PATTERN.equals(ptn);
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/375d4f79/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 6487bfa..11b9bdf 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -96,7 +96,7 @@ public class PartitionDesc {
 return false;
 
 DataType type = partitionDateColumnRef.getType();
-return type.isInt();
+return (type.isInt() || type.isBigInt()) && 
DateFormat.isDatePattern(partitionDateFormat);
 }
 
 public boolean partitionColumnIsTimeMillis() {
@@ -104,7 +104,7 @@ public class PartitionDesc {
 return false;
 
 DataType type = partitionDateColumnRef.getType();
-return type.isBigInt();
+return type.isBigInt() && 
!DateFormat.isDatePattern(partitionDateFormat);
 }
 
 public boolean isPartitioned() {

kylin git commit: minor, update default engines in cloned cube

2016-11-03 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang21 631e6f1c1 -> 7ef9f2ae6


minor, update default engines in cloned cube


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7ef9f2ae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7ef9f2ae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7ef9f2ae

Branch: refs/heads/yang21
Commit: 7ef9f2ae6f1ba0b3fc870dcecc78745461be198f
Parents: 631e6f1
Author: Li Yang 
Authored: Thu Nov 3 16:39:27 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 16:39:39 2016 +0800

--
 .../java/org/apache/kylin/rest/controller/CubeController.java   | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7ef9f2ae/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8dfaed6..64fde81 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -28,6 +28,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
@@ -359,7 +360,11 @@ public class CubeController extends BasicController {
 
 CubeDesc cubeDesc = cube.getDescriptor();
 CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
+
+KylinConfig config = cubeService.getConfig();
 newCubeDesc.setName(newCubeName);
+newCubeDesc.setEngineType(config.getDefaultCubeEngine());
+newCubeDesc.setStorageType(config.getDefaultStorageEngine());
 
 CubeInstance newCube;
 try {

[3/8] kylin git commit: KYLIN-2154 source table rows will be skipped if TOPN's group column contains NULL values

2016-11-03 Thread mahongbin

KYLIN-2154 source table rows will be skipped if TOPN's group column contains 
NULL values


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da2d4d0b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da2d4d0b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da2d4d0b

Branch: refs/heads/yang21-cdh5.7
Commit: da2d4d0b38c8dc1a32c4ee2049954b6a5a684ff3
Parents: f3ac9ff
Author: Hongbin Ma 
Authored: Wed Nov 2 22:28:35 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 10:33:57 2016 +0800

--
 .../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 9 +++--
 .../apache/kylin/storage/gtrecord/CubeSegmentScanner.java   | 5 -
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/da2d4d0b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 3974a4f..1e2d6dd 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.measure.topn;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -142,8 +143,12 @@ public class TopNMeasureType extends 
MeasureType {
 final ByteArray key = new ByteArray(keyLength);
 int offset = 0;
 for (int i = 0; i < dimensionEncodings.length; i++) {
-byte[] valueBytes = Bytes.toBytes(values[i + 1]);
-dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+if (values[i + 1] == null) {
+Arrays.fill(key.array(), offset, offset + 
dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL);
+} else {
+byte[] valueBytes = Bytes.toBytes(values[i + 1]);
+dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+}
 offset += dimensionEncodings[i].getLengthOfEncoding();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/da2d4d0b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 6e19c09..c6a6daa 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -53,6 +53,9 @@ public class CubeSegmentScanner implements IGTScanner {
 
 public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, 
Set dimensions, Set groups, //
 Collection metrics, TupleFilter originalfilter, 
StorageContext context, String gtStorage) {
+
+logger.info("Init CubeSegmentScanner for segment {}", 
cubeSeg.getName());
+
 this.cuboid = cuboid;
 this.cubeSeg = cubeSeg;
 
@@ -61,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner {
 //is working on its own copy
 byte[] serialize = TupleFilterSerializer.serialize(originalfilter, 
StringCodeSystem.INSTANCE);
 TupleFilter filter = TupleFilterSerializer.deserialize(serialize, 
StringCodeSystem.INSTANCE);
-
+
 // translate FunctionTupleFilter to IN clause
 ITupleFilterTransformer translator = new 
BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
 filter = translator.transform(filter);

[7/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a93c2758
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a93c2758
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a93c2758

Branch: refs/heads/yang21-cdh5.7
Commit: a93c2758ad6ca1250708a69746ef38aeb588c3be
Parents: 6be0888
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 14:29:37 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a93c2758/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[5/8] kylin git commit: minor, fix PrintHBaseConfig checkstyle

2016-11-03 Thread mahongbin

minor, fix PrintHBaseConfig checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6be08881
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6be08881
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6be08881

Branch: refs/heads/yang21-cdh5.7
Commit: 6be088815aa41c6a45ce66b28d6e574b681f3f3b
Parents: 9463e37
Author: Li Yang 
Authored: Thu Nov 3 14:14:05 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 14:14:05 2016 +0800

--
 .../java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6be08881/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
index 9ece816..b6c14e3 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
@@ -33,7 +33,7 @@ public class PrintHBaseConfig {
 MyConfig config = new MyConfig(HBaseConfiguration.create());
 
 if (args.length == 0) {
-for (Map.Entry item: config.getProps().entrySet()) {
+for (Map.Entry item : 
config.getProps().entrySet()) {
 System.out.println(item.getKey() + "=" + item.getValue());
 }
 System.exit(0);

[4/8] kylin git commit: minor, add util PrintHBaseConfig

2016-11-03 Thread mahongbin

minor, add util PrintHBaseConfig


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9463e376
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9463e376
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9463e376

Branch: refs/heads/yang21-cdh5.7
Commit: 9463e376c38115a8ecf815e5d610ad4a33c6f3cb
Parents: 2cfc1b8
Author: Li Yang 
Authored: Thu Nov 3 13:50:44 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 13:50:54 2016 +0800

--
 .../storage/hbase/util/PrintHBaseConfig.java| 62 
 1 file changed, 62 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9463e376/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
new file mode 100644
index 000..9ece816
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.util;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+/**
+ */
+public class PrintHBaseConfig {
+
+public static void main(String[] args) throws IOException {
+MyConfig config = new MyConfig(HBaseConfiguration.create());
+
+if (args.length == 0) {
+for (Map.Entry item: config.getProps().entrySet()) {
+System.out.println(item.getKey() + "=" + item.getValue());
+}
+System.exit(0);
+}
+
+if (args.length == 1) {
+System.out.println(config.get(args[0]));
+System.exit(0);
+}
+
+for (String arg : args) {
+System.out.println(arg + "=" + config.get(arg));
+}
+System.exit(0);
+}
+
+private static class MyConfig extends Configuration {
+MyConfig(Configuration other) {
+super(other);
+}
+
+protected synchronized Properties getProps() {
+return super.getProps();
+}
+}
+}

[2/8] kylin git commit: KYLIN-2155 get-properties.sh doesn't support parameters starting with -n (by Ricardo Jorge de Sousa Teixeira)

2016-11-03 Thread mahongbin

KYLIN-2155 get-properties.sh doesn't support parameters starting with -n (by 
Ricardo Jorge de Sousa Teixeira)


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2cfc1b88
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2cfc1b88
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2cfc1b88

Branch: refs/heads/yang21-cdh5.7
Commit: 2cfc1b88d797c7c9c0febd9a55ebe90095203b01
Parents: da2d4d0
Author: Hongbin Ma 
Authored: Thu Nov 3 10:33:34 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 10:33:57 2016 +0800

--
 build/bin/get-properties.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2cfc1b88/build/bin/get-properties.sh
--
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index a53de6b..b0c7387 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -30,4 +30,4 @@ do
:
result=$i
 done
-echo $result
\ No newline at end of file
+echo "$result"
\ No newline at end of file

[6/8] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/a93c2758/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a93c2758/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[1/7] kylin git commit: KYLIN-2141: Add program-friendly interfaces for ResourceTool [Forced Update!]

2016-11-03 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/yang21-hbase1.x 6770b421a -> a93c2758a (forced update)


KYLIN-2141: Add program-friendly interfaces for ResourceTool

Signed-off-by: Yang Li 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f3ac9ff4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f3ac9ff4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f3ac9ff4

Branch: refs/heads/yang21-hbase1.x
Commit: f3ac9ff4f20f5d7e4b28eaff99b388f796594824
Parents: b662a95
Author: Yiming Liu 
Authored: Mon Oct 31 17:45:09 2016 +0800
Committer: Yang Li 
Committed: Wed Nov 2 22:03:51 2016 +0800

--
 .../kylin/common/persistence/ResourceTool.java  | 28 +---
 1 file changed, 24 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f3ac9ff4/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 2f16eb7..2c8bc83 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -49,11 +49,11 @@ public class ResourceTool {
 
 String include = System.getProperty("include");
 if (include != null) {
-includes = include.split("\\s*,\\s*");
+setIncludes(include.split("\\s*,\\s*"));
 }
 String exclude = System.getProperty("exclude");
 if (exclude != null) {
-excludes = exclude.split("\\s*,\\s*");
+setExcludes(exclude.split("\\s*,\\s*"));
 }
 
 String cmd = args[0];
@@ -84,26 +84,46 @@ public class ResourceTool {
 }
 }
 
-public static void cat(KylinConfig config, String path) throws IOException 
{
+public static String[] getIncludes() {
+return includes;
+}
+
+public static void setIncludes(String[] arg) {
+includes = arg;
+}
+
+public static String[] getExcludes() {
+return excludes;
+}
+
+public static void setExcludes(String[] arg) {
+excludes = arg;
+}
+
+public static String cat(KylinConfig config, String path) throws 
IOException {
 ResourceStore store = ResourceStore.getStore(config);
 InputStream is = store.getResource(path).inputStream;
 BufferedReader br = null;
+StringBuffer sb = new StringBuffer();
 String line;
 try {
 br = new BufferedReader(new InputStreamReader(is));
 while ((line = br.readLine()) != null) {
 System.out.println(line);
+sb.append(line).append('\n');
 }
 } finally {
 IOUtils.closeQuietly(is);
 IOUtils.closeQuietly(br);
 }
+return sb.toString();
 }
 
-public static void list(KylinConfig config, String path) throws 
IOException {
+public static NavigableSet list(KylinConfig config, String path) 
throws IOException {
 ResourceStore store = ResourceStore.getStore(config);
 NavigableSet result = store.listResources(path);
 System.out.println("" + result);
+return result;
 }
 
 public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, 
String path) throws IOException {

[5/7] kylin git commit: minor, fix PrintHBaseConfig checkstyle

2016-11-03 Thread mahongbin

minor, fix PrintHBaseConfig checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6be08881
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6be08881
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6be08881

Branch: refs/heads/yang21-hbase1.x
Commit: 6be088815aa41c6a45ce66b28d6e574b681f3f3b
Parents: 9463e37
Author: Li Yang 
Authored: Thu Nov 3 14:14:05 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 14:14:05 2016 +0800

--
 .../java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6be08881/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
index 9ece816..b6c14e3 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
@@ -33,7 +33,7 @@ public class PrintHBaseConfig {
 MyConfig config = new MyConfig(HBaseConfiguration.create());
 
 if (args.length == 0) {
-for (Map.Entry item: config.getProps().entrySet()) {
+for (Map.Entry item : 
config.getProps().entrySet()) {
 System.out.println(item.getKey() + "=" + item.getValue());
 }
 System.exit(0);

[4/7] kylin git commit: minor, add util PrintHBaseConfig

2016-11-03 Thread mahongbin

minor, add util PrintHBaseConfig


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9463e376
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9463e376
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9463e376

Branch: refs/heads/yang21-hbase1.x
Commit: 9463e376c38115a8ecf815e5d610ad4a33c6f3cb
Parents: 2cfc1b8
Author: Li Yang 
Authored: Thu Nov 3 13:50:44 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 13:50:54 2016 +0800

--
 .../storage/hbase/util/PrintHBaseConfig.java| 62 
 1 file changed, 62 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9463e376/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
new file mode 100644
index 000..9ece816
--- /dev/null
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.util;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+/**
+ */
+public class PrintHBaseConfig {
+
+public static void main(String[] args) throws IOException {
+MyConfig config = new MyConfig(HBaseConfiguration.create());
+
+if (args.length == 0) {
+for (Map.Entry item: config.getProps().entrySet()) {
+System.out.println(item.getKey() + "=" + item.getValue());
+}
+System.exit(0);
+}
+
+if (args.length == 1) {
+System.out.println(config.get(args[0]));
+System.exit(0);
+}
+
+for (String arg : args) {
+System.out.println(arg + "=" + config.get(arg));
+}
+System.exit(0);
+}
+
+private static class MyConfig extends Configuration {
+MyConfig(Configuration other) {
+super(other);
+}
+
+protected synchronized Properties getProps() {
+return super.getProps();
+}
+}
+}

[6/7] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

http://git-wip-us.apache.org/repos/asf/kylin/blob/a93c2758/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 0e95102..c59fb33 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput 
{
 try {
 CubeSegment cubeSegment = (CubeSegment) buildable;
 
-final HTableInterface hTable;
+final Table hTable;
 hTable = createHTable(cubeSegment);
 List cuboidWriters = Lists.newArrayList();
 cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements 
IStreamingOutput {
 }
 }
 
-private HTableInterface createHTable(final CubeSegment cubeSegment) throws 
IOException {
+private Table createHTable(final CubeSegment cubeSegment) throws 
IOException {
 final String hTableName = cubeSegment.getStorageLocationIdentifier();
 CubeHTableUtil.createHTable(cubeSegment, null);
-final HTableInterface hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+final Table hTable = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
 logger.info("hTable:" + hTableName + " for segment:" + 
cubeSegment.getName() + " created!");
 return hTable;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a93c2758/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
 List oldTables = getOldHTables();
 if (oldTables != null && oldTables.size() > 0) {
 String metadataUrlPrefix = 
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
-Configuration conf = 
HBaseConnection.getCurrentHBaseConfiguration();
-HBaseAdmin admin = null;
+Admin admin = null;
 try {
-admin = new HBaseAdmin(conf);
+Connection conn = 
HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+admin = conn.getAdmin();
+
 for (String table : oldTables) {
-if (admin.tableExists(table)) {
-HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(Bytes.toBytes(table));
+if (admin.tableExists(TableName.valueOf(table))) {
+HTableDescriptor tableDescriptor = 
admin.getTableDescriptor(TableName.valueOf((table)));
 String host = 
tableDescriptor.getValue(IRealizationConstants.HTableTag);
 if (metadataUrlPrefix.equalsIgnoreCase(host)) {
-if (admin.isTableEnabled(table)) {
-admin.disableTable(table);
+

[7/7] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2016-11-03 Thread mahongbin

KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a93c2758
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a93c2758
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a93c2758

Branch: refs/heads/yang21-hbase1.x
Commit: a93c2758ad6ca1250708a69746ef38aeb588c3be
Parents: 6be0888
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 14:29:37 2016 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 13 ++-
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 36 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../kylin/storage/hbase/HBaseStorage.java   |  3 +-
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 .../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
 .../cube/v1/SerializedHBaseTupleIterator.java   |  4 +-
 .../observer/AggregateRegionObserver.java   |  4 +-
 .../observer/AggregationScanner.java| 14 ++-
 .../observer/ObserverAggregationCache.java  | 10 ++-
 .../coprocessor/observer/ObserverEnabler.java   |  4 +-
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 23 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../hbase/steps/HBaseStreamingOutput.java   |  9 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 36 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 22 ++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../observer/AggregateRegionObserverTest.java   | 26 ++
 .../v1/filter/TestFuzzyRowFilterV2EndToEnd.java |  5 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 36 
 .../kylin/tool/CubeMigrationCheckCLI.java   | 16 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 .../apache/kylin/tool/StorageCleanupJob.java| 20 +++--
 53 files changed, 450 insertions(+), 500 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a93c2758/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-

[3/7] kylin git commit: KYLIN-2154 source table rows will be skipped if TOPN's group column contains NULL values

2016-11-03 Thread mahongbin

KYLIN-2154 source table rows will be skipped if TOPN's group column contains 
NULL values


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da2d4d0b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da2d4d0b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da2d4d0b

Branch: refs/heads/yang21-hbase1.x
Commit: da2d4d0b38c8dc1a32c4ee2049954b6a5a684ff3
Parents: f3ac9ff
Author: Hongbin Ma 
Authored: Wed Nov 2 22:28:35 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Nov 3 10:33:57 2016 +0800

--
 .../java/org/apache/kylin/measure/topn/TopNMeasureType.java | 9 +++--
 .../apache/kylin/storage/gtrecord/CubeSegmentScanner.java   | 5 -
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/da2d4d0b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 3974a4f..1e2d6dd 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.measure.topn;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -142,8 +143,12 @@ public class TopNMeasureType extends 
MeasureType {
 final ByteArray key = new ByteArray(keyLength);
 int offset = 0;
 for (int i = 0; i < dimensionEncodings.length; i++) {
-byte[] valueBytes = Bytes.toBytes(values[i + 1]);
-dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+if (values[i + 1] == null) {
+Arrays.fill(key.array(), offset, offset + 
dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL);
+} else {
+byte[] valueBytes = Bytes.toBytes(values[i + 1]);
+dimensionEncodings[i].encode(valueBytes, 
valueBytes.length, key.array(), offset);
+}
 offset += dimensionEncodings[i].getLengthOfEncoding();
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/da2d4d0b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 6e19c09..c6a6daa 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -53,6 +53,9 @@ public class CubeSegmentScanner implements IGTScanner {
 
 public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, 
Set dimensions, Set groups, //
 Collection metrics, TupleFilter originalfilter, 
StorageContext context, String gtStorage) {
+
+logger.info("Init CubeSegmentScanner for segment {}", 
cubeSeg.getName());
+
 this.cuboid = cuboid;
 this.cubeSeg = cubeSeg;
 
@@ -61,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner {
 //is working on its own copy
 byte[] serialize = TupleFilterSerializer.serialize(originalfilter, 
StringCodeSystem.INSTANCE);
 TupleFilter filter = TupleFilterSerializer.deserialize(serialize, 
StringCodeSystem.INSTANCE);
-
+
 // translate FunctionTupleFilter to IN clause
 ITupleFilterTransformer translator = new 
BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
 filter = translator.transform(filter);

kylin git commit: minor, fix PrintHBaseConfig checkstyle

2016-11-03 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/master 03f385795 -> 50059e388


minor, fix PrintHBaseConfig checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50059e38
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50059e38
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50059e38

Branch: refs/heads/master
Commit: 50059e388ea59c42fb9eeea2d1d92927114bcdbd
Parents: 03f3857
Author: Li Yang 
Authored: Thu Nov 3 14:14:05 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 14:15:03 2016 +0800

--
 .../java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/50059e38/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
index 634ebdf..f9b7daf 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
@@ -33,7 +33,7 @@ public class PrintHBaseConfig {
 MyConfig config = new MyConfig(HBaseConfiguration.create());
 
 if (args.length == 0) {
-for (Map.Entry item: config.getProps().entrySet()) {
+for (Map.Entry item : 
config.getProps().entrySet()) {
 System.out.println(item.getKey() + "=" + item.getValue());
 }
 System.exit(0);

kylin git commit: minor, fix PrintHBaseConfig checkstyle

2016-11-03 Thread liyang

Repository: kylin
Updated Branches:
  refs/heads/yang21 9463e376c -> 6be088815


minor, fix PrintHBaseConfig checkstyle


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6be08881
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6be08881
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6be08881

Branch: refs/heads/yang21
Commit: 6be088815aa41c6a45ce66b28d6e574b681f3f3b
Parents: 9463e37
Author: Li Yang 
Authored: Thu Nov 3 14:14:05 2016 +0800
Committer: Li Yang 
Committed: Thu Nov 3 14:14:05 2016 +0800

--
 .../java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6be08881/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
index 9ece816..b6c14e3 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PrintHBaseConfig.java
@@ -33,7 +33,7 @@ public class PrintHBaseConfig {
 MyConfig config = new MyConfig(HBaseConfiguration.create());
 
 if (args.length == 0) {
-for (Map.Entry item: config.getProps().entrySet()) {
+for (Map.Entry item : 
config.getProps().entrySet()) {
 System.out.println(item.getKey() + "=" + item.getValue());
 }
 System.exit(0);

70 matches

Mail list logo