kylin git commit: KYLIN-2217 use column’s identity as path name and add more logging info

2017-01-21 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2242 63cd178fb -> 73ec2b956


KYLIN-2217 use column’s identity as path name and add more logging info


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73ec2b95
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73ec2b95
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73ec2b95

Branch: refs/heads/KYLIN-2242
Commit: 73ec2b9561597711a0817c7daa03700a3af4fe6a
Parents: 63cd178
Author: shaofengshi 
Authored: Sun Jan 22 15:52:17 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 15:52:17 2017 +0800

--
 .../org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java   | 3 +++
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java| 2 +-
 .../apache/kylin/engine/mr/steps/CreateDictionaryJob.java   | 7 ++-
 .../kylin/engine/mr/steps/FactDistinctColumnsReducer.java   | 9 ++---
 .../kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java | 2 +-
 .../source/hive/cardinality/HiveColumnCardinalityJob.java   | 4 
 6 files changed, 21 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec2b95/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 3e1ab0d..2e5a38e 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -63,11 +63,14 @@ public class DictionaryGeneratorCLI {
 if (dictProvider != null) {
 Dictionary dict = dictProvider.getDictionary(col);
 if (dict != null) {
+logger.debug("Dict for '" + col.getName() + "' has already 
been built, save it");
 cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);
 } else {
+logger.debug("Dict for '" + col.getName() + "' not 
pre-built, build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 } else {
+logger.debug("Dict for '" + col.getName() + "' not pre-built, 
build it from " + inpTable.toString());
 cubeMgr.buildDictionary(cubeSeg, col, inpTable);
 }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec2b95/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 567c1d0..44686d6 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -75,7 +75,7 @@ import org.slf4j.LoggerFactory;
 
 @SuppressWarnings("static-access")
 public abstract class AbstractHadoopJob extends Configured implements Tool {
-protected static final Logger logger = 
LoggerFactory.getLogger(AbstractHadoopJob.class);
+private static final Logger logger = 
LoggerFactory.getLogger(AbstractHadoopJob.class);
 
 protected static final Option OPTION_JOB_NAME = 
OptionBuilder.withArgName(BatchConstants.ARG_JOB_NAME).hasArg().isRequired(true).withDescription("Job
 name. For example, 
Kylin_Cuboid_Builder-clsfd_v2_Step_22-D)").create(BatchConstants.ARG_JOB_NAME);
 protected static final Option OPTION_CUBE_NAME = 
OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube
 name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME);

http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec2b95/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 5b1a7cf..be8c305 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -42,9 +42,13 @@ import org.apache.kylin.engine.mr.SortedColumnDFSFile;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.metadata.model.TblColRef;
 import

kylin git commit: KYLIN-2412 Unclosed DataOutputStream in RoaringBitmapCounter#write()

2017-01-21 Thread gaodayue 
Repository: kylin
Updated Branches:
  refs/heads/master 7576a09f2 -> d264339b1


KYLIN-2412 Unclosed DataOutputStream in RoaringBitmapCounter#write()


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d264339b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d264339b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d264339b

Branch: refs/heads/master
Commit: d264339b1c16c195ffafc2217b793d81bdbd6434
Parents: 7576a09
Author: gaodayue 
Authored: Sun Jan 22 14:02:09 2017 +0800
Committer: gaodayue 
Committed: Sun Jan 22 14:02:09 2017 +0800

--
 .../org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java| 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d264339b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
index cd07d20..fb9dcfc 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java
@@ -107,7 +107,9 @@ public class RoaringBitmapCounter implements BitmapCounter {
 if (out.remaining() < bitmap.serializedSizeInBytes()) {
 throw new BufferOverflowException();
 }
-bitmap.serialize(new DataOutputStream(new 
ByteBufferOutputStream(out)));
+try (DataOutputStream dos = new DataOutputStream(new 
ByteBufferOutputStream(out))) {
+bitmap.serialize(dos);
+}
 }
 
 @Override

kylin git commit: Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs"

2017-01-21 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/master 7de8aa120 -> 7576a09f2


Revert "KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs"

This reverts commit 7de8aa1203a72bad105ed692f7100535939b03af.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7576a09f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7576a09f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7576a09f

Branch: refs/heads/master
Commit: 7576a09f29b75f2eb80a1b5ca9c65fe5c81b1144
Parents: 7de8aa1
Author: shaofengshi 
Authored: Sun Jan 22 11:38:17 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 11:38:17 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 --
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 +--
 .../mr/steps/FactDistinctColumnsReducer.java| 240 ---
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 187 insertions(+), 175 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..bdc4c3e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,10 +26,8 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -142,18 +140,4 @@ public class HadoopUtil {
 }
 }
 
-public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
-FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
-@Override
-public boolean accept(Path path) {
-return path.getName().startsWith(filter);
-}
-});
-
-if (fileStatus.length == 1) {
-return fileStatus[0].getPath();
-} else {
-return null;
-}
-}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index c34a904..696b22a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
+return getRealizationRootPath(jobId) + "/statistics";
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7576a09f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 602b4bb..0281539 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,16 +53,9 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
+String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
-String

Jenkins build is back to stable : Kylin-Master-JDK-1.7 #133

2017-01-21 Thread Apache Jenkins Server 
See

kylin git commit: KYLIN-2242 fix CI

2017-01-21 Thread shaofengshi 
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2242 7de8aa120 -> 63cd178fb


KYLIN-2242 fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/63cd178f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/63cd178f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/63cd178f

Branch: refs/heads/KYLIN-2242
Commit: 63cd178fb3436ba166bb60f0fd08a0bf508382b2
Parents: 7de8aa1
Author: shaofengshi 
Authored: Sun Jan 22 09:43:10 2017 +0800
Committer: shaofengshi 
Committed: Sun Jan 22 09:43:10 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/util/HadoopUtil.java   | 4 
 .../org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java| 2 +-
 .../kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java  | 2 +-
 3 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/63cd178f/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index b9ffe38..f242515 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -143,6 +143,10 @@ public class HadoopUtil {
 }
 
 public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+if (fs.exists(baseDir) == false) {
+return null;
+}
+
 FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
 @Override
 public boolean accept(Path path) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/63cd178f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index e5d053b..5b1a7cf 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -69,7 +69,7 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
 @Override
 public Dictionary getDictionary(TblColRef col) throws 
IOException {
 Path colDir = new Path(factColumnsInputPath, col.getName());
-FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
+FileSystem fs = HadoopUtil.getWorkingFileSystem();
 
 Path dictFile = HadoopUtil.getFilterOnlyPath(fs, colDir, 
col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
 if (dictFile == null) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/63cd178f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 81d5c42..22faa4d 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -81,7 +81,7 @@ public class UpdateCubeInfoAfterBuildStep extends 
AbstractExecutable {
 final TblColRef partitionCol = 
segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
 final String factColumnsInputPath = 
this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
 Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
-FileSystem fs = HadoopUtil.getFileSystem(colDir.toString());
+FileSystem fs = HadoopUtil.getWorkingFileSystem();
 Path outputFile = HadoopUtil.getFilterOnlyPath(fs, colDir, 
partitionCol.getName() + 
FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
 if (outputFile == null) {
 throw new IOException("fail to find the partition file in base 
dir: " + colDir);

kylin git commit: KYLIN-2242 write multiple files in FactDistinctColumnsReducer with MultipleOutputs

2017-01-21 Thread kangkaisen 
Repository: kylin
Updated Branches:
  refs/heads/master c2229c9c7 -> 7de8aa120


KYLIN-2242 write multiple files in FactDistinctColumnsReducer with 
MultipleOutputs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7de8aa12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7de8aa12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7de8aa12

Branch: refs/heads/master
Commit: 7de8aa1203a72bad105ed692f7100535939b03af
Parents: c2229c9
Author: kangkaisen 
Authored: Sat Dec 17 14:12:48 2016 +0800
Committer: kangkaisen 
Committed: Sat Jan 21 23:19:50 2017 +0800

--
 .../apache/kylin/common/util/HadoopUtil.java|  16 ++
 .../kylin/engine/mr/JobBuilderSupport.java  |   2 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   9 +-
 .../engine/mr/steps/CreateDictionaryJob.java|  43 ++--
 .../engine/mr/steps/FactDistinctColumnsJob.java |  32 ++-
 .../mr/steps/FactDistinctColumnsReducer.java| 240 +++
 .../engine/mr/steps/SaveStatisticsStep.java |  10 +-
 .../mr/steps/UpdateCubeInfoAfterBuildStep.java  |  10 +-
 8 files changed, 175 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index bdc4c3e..b9ffe38 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.io.Writable;
 import org.apache.kylin.common.KylinConfig;
 import org.slf4j.Logger;
@@ -140,4 +142,18 @@ public class HadoopUtil {
 }
 }
 
+public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final 
String filter) throws IOException {
+FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+@Override
+public boolean accept(Path path) {
+return path.getName().startsWith(filter);
+}
+});
+
+if (fileStatus.length == 1) {
+return fileStatus[0].getPath();
+} else {
+return null;
+}
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 696b22a..c34a904 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -171,7 +171,7 @@ public class JobBuilderSupport {
 }
 
 public String getStatisticsPath(String jobId) {
-return getRealizationRootPath(jobId) + "/statistics";
+return getRealizationRootPath(jobId) + "/fact_distinct_columns/" + 
BatchConstants.CFG_OUTPUT_STATISTICS;
 }
 
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/7de8aa12/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
--
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 0281539..602b4bb 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -53,9 +53,16 @@ public interface BatchConstants {
 String CFG_STATISTICS_ENABLED = "statistics.enabled";
 String CFG_STATISTICS_OUTPUT = "statistics.ouput";//spell error, for 
compatibility issue better not change it
 String CFG_STATISTICS_SAMPLING_PERCENT = "statistics.sampling.percent";
-String CFG_STATISTICS_CUBE_ESTIMATION_FILENAME = "cube_statistics.txt";
 String CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME = "cuboid_statistics.seq";
 
+String CFG_MAPRED_OUTPUT_COMPRESS = "mapred.output.compress";
+
+String CFG_OUTPUT_COLUMN =

[2/6] kylin git commit: KYLIN-2308 Allow user to set more columnFamily in web

2017-01-21 Thread lidong 
KYLIN-2308 Allow user to set more columnFamily in web

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8331d8d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8331d8d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8331d8d0

Branch: refs/heads/master-cdh5.7
Commit: 8331d8d0ad37d24bdbfa90d1032bcdd063aa0a59
Parents: a853a7c
Author: kangkaisen 
Authored: Sat Jan 7 15:34:57 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 17:25:28 2017 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 75 +++-
 webapp/app/js/controllers/cubeEdit.js   | 51 ---
 webapp/app/js/controllers/cubeSchema.js | 13 +++
 webapp/app/js/filters/filter.js | 22 +++--
 .../cubeDesigner/advanced_settings.html | 92 +++-
 5 files changed, 194 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 760133a..39d36b0 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -278,11 +278,84 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 $scope.isReuse=!$scope.isReuse;
   }
 
-  $scope.removeDictionaries =  function(arr,element){
+  $scope.removeElement =  function(arr,element){
 var index = arr.indexOf(element);
 if (index > -1) {
   arr.splice(index, 1);
 }
   };
 
+  $scope.newColFamily = function (index) {
+return {
+"name": "F" + index,
+"columns": [
+  {
+"qualifier": "M",
+"measure_refs": []
+  }
+]
+  };
+  };
+
+  $scope.initColumnFamily = function () {
+$scope.cubeMetaFrame.hbase_mapping.column_family = [];
+var normalMeasures = [], distinctCountMeasures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  if (measure.function.expression === 'COUNT_DISTINCT') {
+distinctCountMeasures.push(measure);
+  } else {
+normalMeasures.push(measure);
+  }
+});
+if (normalMeasures.length > 0) {
+  var nmcf = $scope.newColFamily(1);
+  angular.forEach(normalMeasures, function (normalM, index) {
+nmcf.columns[0].measure_refs.push(normalM.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(nmcf);
+}
+
+if (distinctCountMeasures.length > 0) {
+  var dccf = $scope.newColFamily(2);
+  angular.forEach(distinctCountMeasures, function (dcm, index) {
+dccf.columns[0].measure_refs.push(dcm.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(dccf);
+}
+  };
+
+  $scope.getAllMeasureNames = function () {
+var measures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  measures.push(measure.name);
+});
+return measures;
+  };
+
+  $scope.getAssignedMeasureNames = function () {
+var assignedMeasures = [];
+angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function 
(colFamily, index) {
+  angular.forEach(colFamily.columns[0].measure_refs, function (measure, 
index) {
+assignedMeasures.push(measure);
+  });
+});
+return assignedMeasures;
+  };
+
+  if ($scope.getAllMeasureNames().length != 
$scope.getAssignedMeasureNames().length) {
+$scope.initColumnFamily();
+  }
+
+
+  $scope.addColumnFamily = function () {
+var colFamily = 
$scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 
1);
+$scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
+  };
+
+  $scope.refreshColumnFamily = function (column_familys, index, colFamily) {
+if (column_familys) {
+  column_familys[index] = colFamily;
+}
+  };
+
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index edbb421..da19b22 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -284,25 +284,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return type;
   };
 
-  var ColFamily = function () {
-var index = 1;
-return function () {
-  var newColFamily =
-  {
-"name": "f" + index,
-"columns": [
-  {
-

[6/6] kylin git commit: KYLIN-1672 support kylin on cdh 5.7

2017-01-21 Thread lidong 
KYLIN-1672 support kylin on cdh 5.7

Signed-off-by: Li Yang 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b2644ee7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b2644ee7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b2644ee7

Branch: refs/heads/master-cdh5.7
Commit: b2644ee70c8967e8dfe767d496096d8c0b4b2c2d
Parents: efbb2ba
Author: Lynne Jiang 
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu 
Committed: Sat Jan 21 21:50:30 2017 +0800

--
 .../kylin/engine/mr/steps/MockupMapContext.java |  13 +-
 examples/test_case_data/sandbox/core-site.xml   | 146 +++---
 examples/test_case_data/sandbox/hbase-site.xml  | 162 ++
 examples/test_case_data/sandbox/hdfs-site.xml   | 259 ++
 examples/test_case_data/sandbox/mapred-site.xml | 398 ++-
 examples/test_case_data/sandbox/yarn-site.xml   | 496 ++-
 pom.xml |  16 +-
 server/pom.xml  |  36 ++
 .../storage/hbase/steps/MockupMapContext.java   |  29 +-
 tool/pom.xml|  12 +
 10 files changed, 430 insertions(+), 1137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b2644ee7/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
--
diff --git 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index dfbba14..bc9601f 100644
--- 
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ 
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -44,10 +44,10 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
 
 /**
  * @author yangli9
- * 
+ *
  */
 @SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext implements MapContext{
+public class MockupMapContext implements MapContext {
 
 private Configuration hconf;
 
@@ -60,7 +60,7 @@ public class MockupMapContext implements MapContext{
 return new WrappedMapper().getMapContext(new MockupMapContext(hconf, 
outKV));
 }
 
-public MockupMapContext(Configuration hconf, Object[] outKV){
+public MockupMapContext(Configuration hconf, Object[] outKV) {
 this.hconf = hconf;
 this.outKV = outKV;
 }
@@ -175,6 +175,11 @@ public class MockupMapContext implements MapContext{
 }
 
 @Override
+public boolean userClassesTakesPrecedence() {
+throw new NotImplementedException();
+}
+
+@Override
 public Class> getInputFormatClass() throws 
ClassNotFoundException {
 throw new NotImplementedException();
 }
@@ -318,4 +323,4 @@ public class MockupMapContext implements MapContext{
 public RawComparator getCombinerKeyGroupingComparator() {
 throw new NotImplementedException();
 }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/b2644ee7/examples/test_case_data/sandbox/core-site.xml
--
diff --git a/examples/test_case_data/sandbox/core-site.xml 
b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
+
 
-
 
 fs.defaultFS
-hdfs://sandbox.hortonworks.com:8020
-true
+hdfs://quickstart.cloudera:8020
 
-
 
 fs.trash.interval
-360
+1
 
-
 
-
ha.failover-controller.active-standby-elector.zk.op.retries
-120
+io.compression.codecs
+
org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec
 
-
 
-hadoop.http.authentication.simple.anonymous.allowed
-true
+hadoop.security.authentication
+simple
 
-
 
-hadoop.proxyuser.falcon.groups
-users
+hadoop.security.authorization
+false
+
+
+hadoop.rpc.protection
+authentication
+
+
+hadoop.security.auth_to_local
+DEFAULT
 
-
 
-hadoop.proxyuser.falcon.hosts
+hadoop.proxyuser.oozie.hosts
 *
 
-
 
-hadoop.proxyuser.hbase.groups
-users
+

[5/6] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-21 Thread lidong 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/efbb2bab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/efbb2bab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/efbb2bab

Branch: refs/heads/master-cdh5.7
Commit: efbb2bab30739bbcb0a9d235ba1a2f34a687cc69
Parents: c2229c9
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Sat Jan 21 21:48:34 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/efbb2bab/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/efbb2bab/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index e02bf19..cbec4f6 100644
---

[5/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-21 Thread lidong 
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/efbb2bab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/efbb2bab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/efbb2bab

Branch: refs/heads/master-hbase1.x
Commit: efbb2bab30739bbcb0a9d235ba1a2f34a687cc69
Parents: c2229c9
Author: shaofengshi 
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu 
Committed: Sat Jan 21 21:48:34 2017 +0800

--
 examples/test_case_data/sandbox/hbase-site.xml  | 19 +---
 .../kylin/provision/BuildCubeWithEngine.java| 19 ++--
 pom.xml | 12 +--
 .../kylin/rest/security/AclHBaseStorage.java|  4 +-
 .../rest/security/MockAclHBaseStorage.java  |  8 +-
 .../apache/kylin/rest/security/MockHTable.java  | 95 
 .../rest/security/RealAclHBaseStorage.java  |  9 +-
 .../apache/kylin/rest/service/AclService.java   | 25 +++---
 .../apache/kylin/rest/service/CubeService.java  | 35 +++-
 .../apache/kylin/rest/service/QueryService.java | 24 +++--
 .../apache/kylin/rest/service/UserService.java  | 17 ++--
 .../kylin/storage/hbase/HBaseConnection.java| 44 -
 .../kylin/storage/hbase/HBaseResourceStore.java | 31 +++
 .../storage/hbase/cube/SimpleHBaseStore.java| 20 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
 .../storage/hbase/cube/v2/CubeHBaseScanRPC.java |  9 +-
 .../coprocessor/endpoint/CubeVisitService.java  |  4 +-
 .../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
 .../storage/hbase/steps/DeprecatedGCStep.java   | 24 ++---
 .../storage/hbase/steps/HBaseCuboidWriter.java  |  7 +-
 .../kylin/storage/hbase/steps/MergeGCStep.java  | 23 ++---
 .../storage/hbase/util/CleanHtableCLI.java  | 12 +--
 .../storage/hbase/util/CubeMigrationCLI.java| 37 
 .../hbase/util/CubeMigrationCheckCLI.java   | 17 ++--
 .../hbase/util/DeployCoprocessorCLI.java| 27 +++---
 .../hbase/util/ExtendCubeToHybridCLI.java   |  8 +-
 .../hbase/util/GridTableHBaseBenchmark.java | 34 +++
 .../kylin/storage/hbase/util/HBaseClean.java| 18 ++--
 .../hbase/util/HBaseRegionSizeCalculator.java   | 35 
 .../kylin/storage/hbase/util/HBaseUsage.java|  9 +-
 .../storage/hbase/util/HbaseStreamingInput.java | 30 +++
 .../hbase/util/HtableAlterMetadataCLI.java  |  9 +-
 .../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
 .../kylin/storage/hbase/util/PingHBaseCLI.java  | 15 ++--
 .../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
 .../storage/hbase/util/StorageCleanupJob.java   | 20 +++--
 .../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
 .../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
 .../kylin/tool/ExtendCubeToHybridCLI.java   |  8 +-
 39 files changed, 366 insertions(+), 437 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/efbb2bab/examples/test_case_data/sandbox/hbase-site.xml
--
diff --git a/examples/test_case_data/sandbox/hbase-site.xml 
b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
 zookeeper.znode.parent
 /hbase-unsecure
 
-
-hbase.client.pause
-100
-General client pause value.  Used mostly as value to wait
-before running a retry of a failed get, region lookup, etc.
-See hbase.client.retries.number for description of how we backoff 
from
-this initial pause amount and how this pause works w/ 
retries.
-
-
-hbase.client.retries.number
-5
-Maximum retries.  Used as maximum for all retryable
-operations such as the getting of a cell's value, starting a row 
update,
-etc.  Retry interval is a rough function based on 
hbase.client.pause.  At
-first we retry at this interval but then with backoff, we pretty 
quickly reach
-retrying every ten seconds.  See HConstants#RETRY_BACKOFF for how 
the backup
-ramps up.  Change this setting and hbase.client.pause to suit your 
workload.
-
+
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/efbb2bab/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java 
b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index e02bf19..cbec4f6 100644
---

[4/5] kylin git commit: KYLIN-1528 Create a branch for v1.5 with HBase 1.x API

2017-01-21 Thread lidong 
http://git-wip-us.apache.org/repos/asf/kylin/blob/efbb2bab/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
 FileSystem fileSystem = FileSystem.get(hconf);
-HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+Admin hbaseAdmin = conn.getAdmin();
 
 String localCoprocessorJar;
 if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
 public static void deployCoprocessor(HTableDescriptor tableDesc) {
 try {
 initHTableCoprocessor(tableDesc);
-logger.info("hbase table " + tableDesc.getName() + " deployed with 
coprocessor.");
+logger.info("hbase table " + tableDesc.getTableName() + " deployed 
with coprocessor.");
 
 } catch (Exception ex) {
-logger.error("Error deploying coprocessor on " + 
tableDesc.getName(), ex);
+logger.error("Error deploying coprocessor on " + 
tableDesc.getTableName(), ex);
 logger.error("Will try creating the table without coprocessor.");
 }
 }
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
 desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
 }
 
-public static boolean resetCoprocessor(String tableName, HBaseAdmin 
hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, 
Path hdfsCoprocessorJar) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 HTableDescriptor desc = 
hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
 
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
 logger.info("reset coprocessor on " + tableName);
 
 logger.info("Disable " + tableName);
-hbaseAdmin.disableTable(tableName);
+hbaseAdmin.disableTable(TableName.valueOf(tableName));
 
 while (desc.hasCoprocessor(CubeObserverClassOld2)) {
 desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
 desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
 }
 
-hbaseAdmin.modifyTable(tableName, desc);
+hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
 
 logger.info("Enable " + tableName);
-hbaseAdmin.enableTable(tableName);
+hbaseAdmin.enableTable(TableName.valueOf(tableName));
 
 return true;
 }
 
-
-private static List resetCoprocessorOnHTables(final HBaseAdmin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
+private static List resetCoprocessorOnHTables(final Admin 
hbaseAdmin, final Path hdfsCoprocessorJar, List tableNames) throws 
IOException {
 List processedTables = Collections.synchronizedList(new 
ArrayList());
 ExecutorService coprocessorPool = 
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
 CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
 
 private static class ResetCoprocessorWorker implements Runnable {
 private final CountDownLatch countDownLatch;
-private final HBaseAdmin hbaseAdmin;
+private final Admin hbaseAdmin;
 private final Path hdfsCoprocessorJar;
 private final String tableName;
 private final List processedTables;
 
-public ResetCoprocessorWorker(CountDownLatch countDownLatch, 
HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List 
processedTables) {
+public ResetCoprocessorWorker(CountDownLatch

[3/5] kylin git commit: KYLIN-2396 remove unused code

2017-01-21 Thread lidong 
KYLIN-2396 remove unused code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2229c9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2229c9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2229c9c

Branch: refs/heads/master-hbase1.x
Commit: c2229c9c7f83fe5a28e564c37ada5794a67ee3d8
Parents: 8331d8d
Author: lidongsjtu 
Authored: Sat Jan 21 19:52:42 2017 +0800
Committer: lidongsjtu 
Committed: Sat Jan 21 19:52:42 2017 +0800

--
 .../measure/percentile/PercentileContUdf.java   | 37 
 .../kylin/rest/controller/BasicController.java  | 22 
 .../rest/controller/DiagnosisController.java| 19 --
 3 files changed, 22 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2229c9c/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
deleted file mode 100644
index 4ef6b75..000
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.percentile;
-
-public class PercentileContUdf {
-public static double init() {
-return 0;
-}
-
-public static double add(double accumulator, double v, double r) {
-return 0;
-}
-
-public static double merge(double accumulator0, double accumulator1) {
-return 0;
-}
-
-public static double result(long accumulator) {
-return 0;
-}
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2229c9c/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
index f61492d..6b4c9fd 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
@@ -18,11 +18,20 @@
 
 package org.apache.kylin.rest.controller;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.exception.NotFoundException;
 import org.apache.kylin.rest.response.ErrorResponse;
 import org.slf4j.Logger;
@@ -74,4 +83,17 @@ public class BasicController {
 }
 }
 
+protected void setDownloadResponse(String downloadFile, final 
HttpServletResponse response) {
+File file = new File(downloadFile);
+try (InputStream fileInputStream = new FileInputStream(file); 
OutputStream output = response.getOutputStream();) {
+response.reset();
+response.setContentType("application/octet-stream");
+response.setContentLength((int) (file.length()));
+response.setHeader("Content-Disposition", "attachment; 
filename=\"" + file.getName() + "\"");
+IOUtils.copyLarge(fileInputStream, output);
+output.flush();
+} catch (IOException e) {
+throw new InternalErrorException("Failed to download file: " + 
e.getMessage(), e);
+}
+}
 }

[1/5] kylin git commit: KYLIN 2370 Refine unload and reload table [Forced Update!]

2017-01-21 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/master-hbase1.x 5feb9e928 -> efbb2bab3 (forced update)


KYLIN 2370 Refine unload and reload table

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a853a7c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a853a7c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a853a7c4

Branch: refs/heads/master-hbase1.x
Commit: a853a7c4b6b7d42cc3ffd066b618f5ea72ca0d9f
Parents: 85a1eb3
Author: chenzhx <346839...@qq.com>
Authored: Mon Jan 9 15:09:16 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 16:21:11 2017 +0800

--
 webapp/app/css/AdminLTE.css |  27 +++
 webapp/app/js/controllers/sourceMeta.js | 175 ---
 .../js/directives/kylin_abn_tree_directive.js   |   7 +-
 .../app/partials/tables/source_table_tree.html  |   6 +-
 webapp/app/partials/tables/table_unload.html|  33 
 5 files changed, 111 insertions(+), 137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/css/AdminLTE.css
--
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index 857dbf7..c7740d1 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -4800,3 +4800,30 @@ Gradient Background colors
 white-space: normal !important;
   }
 }
+
+.abn-tree .abn-tree-row .tree-table-btn{
+width: 4%;
+float: right;
+border-radius: 5px;
+margin-right: 5px;
+padding: 3px 16px 2px 5px;
+color: #ff;
+position:static;
+}
+.abn-tree .abn-tree-row .tree-table-btn .tooltip{
+overflow:visible;
+}
+.abn-tree .abn-tree-row .btn-info:hover{
+   background-color:#269abc;
+}
+.abn-tree > .abn-tree-row.active > .btn-info{
+background-color:#269abc;
+border-left-color:#269abc;
+}
+.abn-tree .abn-tree-row .btn-success:hover{
+background-color:#008d4c;
+}
+.abn-tree > .abn-tree-row.active > .btn-success{
+background-color:#008d4c;
+border-left-color:#008d4c;
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/a853a7c4/webapp/app/js/controllers/sourceMeta.js
--
diff --git a/webapp/app/js/controllers/sourceMeta.js 
b/webapp/app/js/controllers/sourceMeta.js
index a53a35f..5421673 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -132,31 +132,83 @@ KylinApp
   });
 };
 
-$scope.openUnLoadModal = function () {
-  if(!$scope.projectModel.selectedProject){
-SweetAlert.swal('Oops...', "Please select a project.", 'info');
+$scope.reloadTable = function (tableName,projectName){
+  loadingRequest.show();
+  TableService.loadHiveTable({tableName: tableName, action: projectName}, 
{calculate: $scope.isCalculate}, function (result) {
+var loadTableInfo = "";
+angular.forEach(result['result.loaded'], function (table) {
+  loadTableInfo += "\n" + table;
+})
+var unloadedTableInfo = "";
+angular.forEach(result['result.unloaded'], function (table) {
+  unloadedTableInfo += "\n" + table;
+})
+if (result['result.unloaded'].length != 0 && 
result['result.loaded'].length == 0) {
+  SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + 
unloadedTableInfo, 'error');
+}
+if (result['result.loaded'].length != 0 && 
result['result.unloaded'].length == 0) {
+  SweetAlert.swal('Success!', 'The following table(s) have been 
successfully loaded: ' + loadTableInfo, 'success');
+}
+if (result['result.loaded'].length != 0 && 
result['result.unloaded'].length != 0) {
+  SweetAlert.swal('Partial loaded!', 'The following table(s) have been 
successfully loaded: ' + loadTableInfo + "\n\n Failed to load following 
table(s):" + unloadedTableInfo, 'warning');
+}
+loadingRequest.hide();
+$scope.aceSrcTbLoaded(true);
+  }, function (e) {
+if (e.data && e.data.exception) {
+  var message = e.data.exception;
+  var msg = !!(message) ? message : 'Failed to take action.';
+  SweetAlert.swal('Oops...', msg, 'error');
+} else {
+  SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+}
+loadingRequest.hide();
+  })
+}
+
+
+
+$scope.removeList = function (tableName,projectName) {
+  if (tableName.trim() === "") {
+SweetAlert.swal('', 'Please input table(s) you want to unload.', 
'info');
 return;
   }
-  $modal.open({
-templateUrl: 'removeHiveTable.html',
-controller: ModalInstanceCtrl,
-

[2/5] kylin git commit: KYLIN-2308 Allow user to set more columnFamily in web

2017-01-21 Thread lidong 
KYLIN-2308 Allow user to set more columnFamily in web

Signed-off-by: zhongjian 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8331d8d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8331d8d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8331d8d0

Branch: refs/heads/master-hbase1.x
Commit: 8331d8d0ad37d24bdbfa90d1032bcdd063aa0a59
Parents: a853a7c
Author: kangkaisen 
Authored: Sat Jan 7 15:34:57 2017 +0800
Committer: zhongjian 
Committed: Fri Jan 20 17:25:28 2017 +0800

--
 webapp/app/js/controllers/cubeAdvanceSetting.js | 75 +++-
 webapp/app/js/controllers/cubeEdit.js   | 51 ---
 webapp/app/js/controllers/cubeSchema.js | 13 +++
 webapp/app/js/filters/filter.js | 22 +++--
 .../cubeDesigner/advanced_settings.html | 92 +++-
 5 files changed, 194 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeAdvanceSetting.js
--
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js 
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 760133a..39d36b0 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -278,11 +278,84 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function 
($scope, $modal,cubeConfi
 $scope.isReuse=!$scope.isReuse;
   }
 
-  $scope.removeDictionaries =  function(arr,element){
+  $scope.removeElement =  function(arr,element){
 var index = arr.indexOf(element);
 if (index > -1) {
   arr.splice(index, 1);
 }
   };
 
+  $scope.newColFamily = function (index) {
+return {
+"name": "F" + index,
+"columns": [
+  {
+"qualifier": "M",
+"measure_refs": []
+  }
+]
+  };
+  };
+
+  $scope.initColumnFamily = function () {
+$scope.cubeMetaFrame.hbase_mapping.column_family = [];
+var normalMeasures = [], distinctCountMeasures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  if (measure.function.expression === 'COUNT_DISTINCT') {
+distinctCountMeasures.push(measure);
+  } else {
+normalMeasures.push(measure);
+  }
+});
+if (normalMeasures.length > 0) {
+  var nmcf = $scope.newColFamily(1);
+  angular.forEach(normalMeasures, function (normalM, index) {
+nmcf.columns[0].measure_refs.push(normalM.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(nmcf);
+}
+
+if (distinctCountMeasures.length > 0) {
+  var dccf = $scope.newColFamily(2);
+  angular.forEach(distinctCountMeasures, function (dcm, index) {
+dccf.columns[0].measure_refs.push(dcm.name);
+  });
+  $scope.cubeMetaFrame.hbase_mapping.column_family.push(dccf);
+}
+  };
+
+  $scope.getAllMeasureNames = function () {
+var measures = [];
+angular.forEach($scope.cubeMetaFrame.measures, function (measure, index) {
+  measures.push(measure.name);
+});
+return measures;
+  };
+
+  $scope.getAssignedMeasureNames = function () {
+var assignedMeasures = [];
+angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function 
(colFamily, index) {
+  angular.forEach(colFamily.columns[0].measure_refs, function (measure, 
index) {
+assignedMeasures.push(measure);
+  });
+});
+return assignedMeasures;
+  };
+
+  if ($scope.getAllMeasureNames().length != 
$scope.getAssignedMeasureNames().length) {
+$scope.initColumnFamily();
+  }
+
+
+  $scope.addColumnFamily = function () {
+var colFamily = 
$scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 
1);
+$scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
+  };
+
+  $scope.refreshColumnFamily = function (column_familys, index, colFamily) {
+if (column_familys) {
+  column_familys[index] = colFamily;
+}
+  };
+
 });

http://git-wip-us.apache.org/repos/asf/kylin/blob/8331d8d0/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index edbb421..da19b22 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -284,25 +284,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 return type;
   };
 
-  var ColFamily = function () {
-var index = 1;
-return function () {
-  var newColFamily =
-  {
-"name": "f" + index,
-"columns": [
-  {
-

kylin git commit: KYLIN-2396 remove unused code

2017-01-21 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/master 8331d8d0a -> c2229c9c7


KYLIN-2396 remove unused code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2229c9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2229c9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2229c9c

Branch: refs/heads/master
Commit: c2229c9c7f83fe5a28e564c37ada5794a67ee3d8
Parents: 8331d8d
Author: lidongsjtu 
Authored: Sat Jan 21 19:52:42 2017 +0800
Committer: lidongsjtu 
Committed: Sat Jan 21 19:52:42 2017 +0800

--
 .../measure/percentile/PercentileContUdf.java   | 37 
 .../kylin/rest/controller/BasicController.java  | 22 
 .../rest/controller/DiagnosisController.java| 19 --
 3 files changed, 22 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c2229c9c/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
deleted file mode 100644
index 4ef6b75..000
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileContUdf.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.percentile;
-
-public class PercentileContUdf {
-public static double init() {
-return 0;
-}
-
-public static double add(double accumulator, double v, double r) {
-return 0;
-}
-
-public static double merge(double accumulator0, double accumulator1) {
-return 0;
-}
-
-public static double result(long accumulator) {
-return 0;
-}
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/c2229c9c/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
index f61492d..6b4c9fd 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/BasicController.java
@@ -18,11 +18,20 @@
 
 package org.apache.kylin.rest.controller;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.exception.NotFoundException;
 import org.apache.kylin.rest.response.ErrorResponse;
 import org.slf4j.Logger;
@@ -74,4 +83,17 @@ public class BasicController {
 }
 }
 
+protected void setDownloadResponse(String downloadFile, final 
HttpServletResponse response) {
+File file = new File(downloadFile);
+try (InputStream fileInputStream = new FileInputStream(file); 
OutputStream output = response.getOutputStream();) {
+response.reset();
+response.setContentType("application/octet-stream");
+response.setContentLength((int) (file.length()));
+response.setHeader("Content-Disposition", "attachment; 
filename=\"" + file.getName() + "\"");
+IOUtils.copyLarge(fileInputStream, output);
+output.flush();
+} catch (IOException e) {
+throw new InternalErrorException("Failed to download file: " +