[42/50] [abbrv] kylin git commit: KYLIN-2773: fix join condition push down issue caused by type casting

[liyang]

KYLIN-2773: fix join condition push down issue caused by type casting


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87f1732b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87f1732b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87f1732b

Branch: refs/heads/master
Commit: 87f1732b0a26191122788baaa2fc9d97c96210da
Parents: 28ac7d9
Author: Zhong 
Authored: Tue Aug 29 17:19:47 2017 +0800
Committer: Roger Shi 
Committed: Wed Aug 30 16:02:26 2017 +0800

--
 .../calcite/sql2rel/SqlToRelConverter.java  | 54 
 1 file changed, 54 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/87f1732b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index f10ba96..3615df7 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -2468,6 +2468,10 @@ public class SqlToRelConverter {
   return corr;
 }
 
+if (containOnlyCast(joinCond)) {
+  joinCond = convertCastCondition(joinCond);
+}
+
 final Join originalJoin =
 (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel,
 joinCond, ImmutableSet.of(), joinType, false);
@@ -2475,6 +2479,56 @@ public class SqlToRelConverter {
 return RelOptUtil.pushDownJoinConditions(originalJoin);
   }
 
+  private boolean containOnlyCast(RexNode node) {
+boolean result = true;
+switch (node.getKind()) {
+  case AND:
+  case EQUALS:
+final RexCall call = (RexCall) node;
+List operands = Lists.newArrayList(call.getOperands());
+for (int i = 0; i < operands.size(); i++) {
+  RexNode operand = operands.get(i);
+  result &= containOnlyCast(operand);
+}
+break;
+  case OR:
+  case INPUT_REF:
+  case LITERAL:
+  case CAST:
+return true;
+  default:
+return false;
+}
+return result;
+  }
+
+  private static RexNode convertCastCondition(RexNode node) {
+switch (node.getKind()) {
+  case AND:
+  case EQUALS:
+RexCall call = (RexCall) node;
+List list = Lists.newArrayList();
+List operands = Lists.newArrayList(call.getOperands());
+for (int i = 0; i < operands.size(); i++) {
+  RexNode operand = operands.get(i);
+  final RexNode e =
+  convertCastCondition(
+  operand);
+  list.add(e);
+}
+if (!list.equals(call.getOperands())) {
+  return call.clone(call.getType(), list);
+}
+return call;
+  case CAST:
+call = (RexCall) node;
+operands = Lists.newArrayList(call.getOperands());
+return operands.get(0);
+  default:
+return node;
+}
+  }
+
   private CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0) {
 final Set correlatedVariables =
 RelOptUtil.getVariablesUsed(r0);

[49/50] [abbrv] kylin git commit: Revert "#949 Refine column family for measures"

[liyang]

Revert "#949 Refine column family for measures"

This reverts commit 906b8f40b381ab9086a4f7e7593ae7d8391189bd.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32d346b4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32d346b4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32d346b4

Branch: refs/heads/master
Commit: 32d346b4cd619a38461a27115fbef2bd44f28372
Parents: b50030d
Author: Hongbin Ma 
Authored: Thu Aug 31 22:41:20 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Aug 31 22:41:20 2017 +0800

--
 .../org/apache/kylin/gridtable/GTRecord.java | 19 +--
 1 file changed, 13 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/32d346b4/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 00e355f..d8c0fbd 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -45,7 +45,7 @@ public class GTRecord implements Comparable, 
Cloneable {
 }
 this.info = info;
 }
-
+
 @Override
 public GTRecord clone() { // deep copy
 ByteArray[] cols = new ByteArray[this.cols.length];
@@ -114,6 +114,7 @@ public class GTRecord implements Comparable, 
Cloneable {
 return result;
 }
 
+
 /** decode and return the values of this record */
 public Object[] getValues(int[] selectedColumns, Object[] result) {
 assert selectedColumns.length <= result.length;
@@ -294,21 +295,27 @@ public class GTRecord implements Comparable, 
Cloneable {
 buf.position(pos);
 }
 }
-
-public void loadColumnsFromColumnBlocks(ImmutableBitSet[] 
selectedColumnBlocks, ImmutableBitSet selectedCols,
-ByteBuffer buf) {
+
+public void loadColumnsFromColumnBlocks(ImmutableBitSet[] 
selectedColumnBlocks, ImmutableBitSet selectedCols, ByteBuffer buf) {
+
 int pos = buf.position();
+
 for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) {
+
 for (int i = 0; i < selectedColBlock.trueBitCount(); i++) {
+
 int c = selectedColBlock.trueBitAt(i);
+
 int len = info.codeSystem.codeLength(c, buf);
-if (selectedCols.get(c)) {
-cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
+if(selectedCols.get(c)) {  
+cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
 
 }
 pos += len;
 buf.position(pos);
 }
+
 }
+
 }
 
 /** change pointers to point to data in given buffer, this

[46/50] [abbrv] kylin git commit: #949 Refine column family for measures

[liyang]

#949 Refine column family for measures


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a4aeb778
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a4aeb778
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a4aeb778

Branch: refs/heads/master
Commit: a4aeb77890577e46300bc702c1117bf542081cc9
Parents: d62cb58
Author: auphyroc99 <454530...@qq.com>
Authored: Wed Aug 30 16:03:24 2017 +0800
Committer: Roger Shi 
Committed: Wed Aug 30 18:13:52 2017 +0800

--
 .../org/apache/kylin/gridtable/GTRecord.java | 19 ++-
 1 file changed, 6 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a4aeb778/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 862fb99..c3a160a 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -45,7 +45,7 @@ public class GTRecord implements Comparable, 
Cloneable {
 }
 this.info = info;
 }
-
+
 @Override
 public GTRecord clone() { // deep copy
 ByteArray[] cols = new ByteArray[this.cols.length];
@@ -114,7 +114,6 @@ public class GTRecord implements Comparable, 
Cloneable {
 return result;
 }
 
-
 /** decode and return the values of this record */
 public Object[] getValues(int[] selectedColumns, Object[] result) {
 assert selectedColumns.length <= result.length;
@@ -295,27 +294,21 @@ public class GTRecord implements Comparable, 
Cloneable {
 buf.position(pos);
 }
 }
-
-public void loadColumnsFromColumnBlocks(ImmutableBitSet[] 
selectedColumnBlocks, ImmutableBitSet selectedCols, ByteBuffer buf) {
-
+
+public void loadColumnsFromColumnBlocks(ImmutableBitSet[] 
selectedColumnBlocks, ImmutableBitSet selectedCols,
+ByteBuffer buf) {
 int pos = buf.position();
-
 for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) {
-
 for (int i = 0; i < selectedColBlock.trueBitCount(); i++) {
-
 int c = selectedColBlock.trueBitAt(i);
-
 int len = info.codeSystem.codeLength(c, buf);
-if(selectedCols.get(c)) {  
-cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
 
+if (selectedCols.get(c)) {
+cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
 }
 pos += len;
 buf.position(pos);
 }
-
 }
-
 }
 
 }

[38/50] [abbrv] kylin git commit: KYLIN-2803 Pushdown non select query

[liyang]

KYLIN-2803 Pushdown non select query


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dbecb9b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dbecb9b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dbecb9b8

Branch: refs/heads/master
Commit: dbecb9b84bc90049acb3ef314128ac71f3516731
Parents: 480592d
Author: shi shaofeng 
Authored: Fri Aug 25 16:18:21 2017 +0800
Committer: SHAOFENG SHI 
Committed: Mon Aug 28 16:02:43 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 ++
 .../adhocquery/HivePushDownConverter.java   |  4 ++
 .../source/adhocquery/IPushDownRunner.java  | 13 ++
 .../query/adhoc/PushDownRunnerJdbcImpl.java | 36 -
 .../apache/kylin/query/util/PushDownUtil.java   | 32 ++-
 .../org/apache/kylin/query/util/QueryUtil.java  | 28 +
 .../apache/kylin/query/util/QueryUtilTest.java  | 39 ++
 .../apache/kylin/rest/service/QueryService.java | 42 +---
 8 files changed, 172 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index bca00e7..a113327 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1088,6 +1088,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 new String[] { 
"org.apache.kylin.source.adhocquery.HivePushDownConverter" });
 }
 
+public boolean isPushdownQueryCacheEnabled() {
+return 
Boolean.parseBoolean(this.getOptional("kylin.query.pushdown.cache-enabled", 
"false"));
+}
+
 public String getJdbcUrl() {
 return getOptional("kylin.query.pushdown.jdbc.url", "");
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
 
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
index bcd8608..eef4594 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HivePushDownConverter.java
@@ -262,6 +262,10 @@ public class HivePushDownConverter implements 
IPushDownConverter {
 // Step7.Add quote for interval in timestampadd
 convertedSql = timestampaddReplace(convertedSql);
 
+// Step8.Replace integer with int
+convertedSql = replaceString(convertedSql, "INTEGER", "INT");
+convertedSql = replaceString(convertedSql, "integer", "int");
+
 return convertedSql;
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
 
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
index c8d18aa..0336bfb 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
@@ -36,4 +36,17 @@ public interface IPushDownRunner {
  * @throws Exception if running pushdown query fails
  */
 void executeQuery(String query, List returnRows, 
List returnColumnMeta) throws Exception;
+
+
+/**
+ * Run an pushdown non-query sql
+ *
+ * @param sql the sql statement
+ *
+ * @return whether the SQL is executed successfully
+ *
+ * @throws Exception if running pushdown fails
+ */
+boolean executeUpdate(String sql) throws Exception;
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/dbecb9b8/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java 
b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
index 751e6b0..713629a 100644
--- 

[50/50] [abbrv] kylin git commit: Revert "KYLIN-2790 Add extending point for column family"

[liyang]

Revert "KYLIN-2790 Add extending point for column family"

This reverts commit 842dc4ca1dabb576aed40c55ebfc1cbabaaf3944.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/363f858b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/363f858b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/363f858b

Branch: refs/heads/master
Commit: 363f858b890ea04c87ee7b081fc74805766e0b93
Parents: 32d346b
Author: Hongbin Ma 
Authored: Thu Aug 31 22:43:36 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Aug 31 22:43:36 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 
 .../org/apache/kylin/cube/model/CubeDesc.java   | 18 ++--
 .../kylin/cube/model/HBaseMappingDesc.java  | 18 
 .../org/apache/kylin/gridtable/GTRecord.java| 22 
 4 files changed, 2 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/363f858b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d32f08c..0e990df 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -286,10 +286,6 @@ abstract public class KylinConfigBase implements 
Serializable {
 
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
 return (DistributedLockFactory) ClassUtil.newInstance(clsName);
 }
-
-public String getHBaseMappingAdapter() {
-return getOptional("kylin.metadata.hbasemapping-adapter");
-}
 
 // 

 // DICTIONARY & SNAPSHOT

http://git-wip-us.apache.org/repos/asf/kylin/blob/363f858b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index d263e1f..0e22587 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -22,7 +22,6 @@ import static 
com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 
-import java.lang.reflect.Method;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
@@ -604,21 +603,8 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 validateAggregationGroups(); // check if aggregation group is valid
 validateAggregationGroupsCombination();
 
-String hbaseMappingAdapterName = config.getHBaseMappingAdapter();
-
-if (hbaseMappingAdapterName != null) {
-try {
-Class hbaseMappingAdapterClass = 
Class.forName(hbaseMappingAdapterName);
-Method initMethod = 
hbaseMappingAdapterClass.getMethod("initHBaseMapping", CubeDesc.class);
-initMethod.invoke(null, this);
-} catch (Exception e) {
-logger.error("Wrong configuration for 
kylin.metadata.hbasemapping-adapter: class "
-+ hbaseMappingAdapterName + " not found. ");
-}
-} else {
-if (hbaseMapping != null) {
-hbaseMapping.init(this);
-}
+if (hbaseMapping != null) {
+hbaseMapping.init(this);
 }
 
 initMeasureReferenceToColumnFamily();

http://git-wip-us.apache.org/repos/asf/kylin/blob/363f858b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index 77b4b14..d1e5829 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -87,24 +87,6 @@ public class HBaseMappingDesc implements 
java.io.Serializable {
 }
 }
 
-public void initAsSeparatedColumns(CubeDesc cubeDesc) {
-cubeRef = cubeDesc;
-
-int cfNum = cubeDesc.getMeasures().size();
-columnFamily = new 

[40/50] [abbrv] kylin git commit: Revert "KYLIN-2816 wrap columns with backtick"

[liyang]

Revert "KYLIN-2816 wrap columns with backtick"

This reverts commit 53b25c6137731c1b0d26925be7dcaa0252d75b64.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81e9f573
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81e9f573
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81e9f573

Branch: refs/heads/master
Commit: 81e9f573fae2af9511ac6004f69101598fc7a71a
Parents: dbecb9b
Author: Hongbin Ma 
Authored: Mon Aug 28 19:19:10 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Aug 29 10:36:10 2017 +0800

--
 .../org/apache/kylin/job/JoinedFlatTable.java   | 77 ++--
 .../kylin/job/TestWrapColumnWithBacktick.java   | 57 ---
 .../metadata/model/tool/CalciteParser.java  |  8 +-
 3 files changed, 7 insertions(+), 135 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/81e9f573/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 4ac5a3a..824c693 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -21,17 +21,11 @@ package org.apache.kylin.job;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.calcite.sql.SqlIdentifier;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.util.SqlBasicVisitor;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.metadata.model.DataModelDesc;
@@ -41,9 +35,6 @@ import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.PartitionDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  *
@@ -52,7 +43,6 @@ import org.slf4j.LoggerFactory;
 public class JoinedFlatTable {
 
 public static final String BACKTICK = "`";
-private static final Logger logger = 
LoggerFactory.getLogger(JoinedFlatTable.class);
 
 public static String getTableDir(IJoinedFlatTableDesc flatDesc, String 
storageDfsDir) {
 return storageDfsDir + "/" + flatDesc.getTableName();
@@ -140,7 +130,11 @@ public class JoinedFlatTable {
 sql.append(",");
 }
 String colTotalName = String.format("%s.%s", 
col.getTableRef().getTableName(), col.getName());
-String expressionInSourceDB = 
quoteOriginalColumnWithBacktick(col.getExpressionInSourceDB());
+String expressionInSourceDB = col.getExpressionInSourceDB();
+if (expressionInSourceDB.contains(".")) {
+// surround column name with back-tick, to support unicode 
column name
+expressionInSourceDB = expressionInSourceDB.replace(".", "." + 
BACKTICK) + BACKTICK;
+}
 if (skipAsList.contains(colTotalName)) {
 sql.append(expressionInSourceDB + sep);
 } else {
@@ -283,65 +277,4 @@ public class JoinedFlatTable {
 return sql.toString();
 }
 
-/**
- * quote column name with back-tick, to support unicode column name
- *
- * @param sourceSQL
- * @return
- */
-static String quoteOriginalColumnWithBacktick(String sourceSQL) {
-StringBuilder result = new StringBuilder(sourceSQL);
-try {
-List> replacePoses = new ArrayList<>();
-for (SqlIdentifier id : 
SqlIdentifierVisitor.getAllSqlIdentifier(CalciteParser.getExpNode(sourceSQL))) {
-
replacePoses.add(CalciteParser.getReplacePos(id.getComponentParserPosition(1), 
"select " + sourceSQL + " from t"));
-}
-
-// latter replace position in the front of the list.
-Collections.sort(replacePoses, new Comparator>() {
-@Override
-public int compare(Pair o1, Pair o2) {
-return -(o1.getSecond() - o2.getSecond());
-}
-});
-
-// cuz the pos is add "select "'s size, so minus 7 offset.
-final int OFFSET = 7;
-for (Pair replacePos : 

[44/50] [abbrv] kylin git commit: KYLIN-2823 Trim TupleFilter after dictionary-based filter optimization

[liyang]

KYLIN-2823 Trim TupleFilter after dictionary-based filter optimization


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47822891
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47822891
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47822891

Branch: refs/heads/master
Commit: 47822891332aef146ae3c2b5e3adc98c23024770
Parents: 1eb3617
Author: Hongbin Ma 
Authored: Wed Aug 30 16:06:18 2017 +0800
Committer: 成 
Committed: Wed Aug 30 17:40:03 2017 +0800

--
 .../java/org/apache/kylin/gridtable/GTUtil.java |  24 +++-
 .../metadata/filter/ConstantTupleFilter.java|  14 +++
 .../filter/FilterOptimizeTransformer.java   |  35 +-
 .../filter/FilterOptimizeTransformerTest.java   | 121 +++
 .../storage/gtrecord/DictGridTableTest.java | 108 +++--
 .../src/test/resources/query/sql/query106.sql   | 101 
 .../src/test/resources/query/sql/query107.sql   | 102 
 7 files changed, 459 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/47822891/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 7a7e4e6..5c9dfe3 100755
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -30,6 +30,7 @@ import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
 import org.apache.kylin.metadata.filter.IFilterCodeSystem;
 import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilterSerializer;
@@ -64,7 +65,15 @@ public class GTUtil {
 public static TupleFilter convertFilterColumnsAndConstants(TupleFilter 
rootFilter, GTInfo info, //
 List colMapping, Set 
unevaluatableColumnCollector) {
 Map map = colListToMap(colMapping);
-return convertFilter(rootFilter, info, map, true, 
unevaluatableColumnCollector);
+TupleFilter filter = convertFilter(rootFilter, info, map, true, 
unevaluatableColumnCollector);
+
+// optimize the filter: after translating with dictionary, some 
filters become determined
+// e.g.
+// ( a = 'value_in_dict' OR a = 'value_not_in_dict') will become (a = 
'value_in_dict' OR ConstantTupleFilter.FALSE)
+// use the following to further trim the filter to (a = 
'value_in_dict')
+// The goal is to avoid too many children after flatten filter step
+filter = new FilterOptimizeTransformer().transform(filter);
+return filter;
 }
 
 protected static Map colListToMap(List 
colMapping) {
@@ -81,8 +90,9 @@ public class GTUtil {
 final Set unevaluatableColumnCollector) {
 
 IFilterCodeSystem filterCodeSystem = 
wrap(info.codeSystem.getComparator());
-
-GTConvertDecorator decorator = new 
GTConvertDecorator(unevaluatableColumnCollector, colMapping, info, 
encodeConstants);
+
+GTConvertDecorator decorator = new 
GTConvertDecorator(unevaluatableColumnCollector, colMapping, info,
+encodeConstants);
 
 byte[] bytes = TupleFilterSerializer.serialize(rootFilter, decorator, 
filterCodeSystem);
 return TupleFilterSerializer.deserialize(bytes, filterCodeSystem);
@@ -122,14 +132,15 @@ public class GTUtil {
 protected final GTInfo info;
 protected final boolean encodeConstants;
 
-public GTConvertDecorator(Set unevaluatableColumnCollector, 
Map colMapping, GTInfo info, boolean encodeConstants) {
+public GTConvertDecorator(Set unevaluatableColumnCollector, 
Map colMapping,
+GTInfo info, boolean encodeConstants) {
 this.unevaluatableColumnCollector = unevaluatableColumnCollector;
 this.colMapping = colMapping;
 this.info = info;
 this.encodeConstants = encodeConstants;
 buf = ByteBuffer.allocate(info.getMaxColumnLength());
 }
-
+
 protected int mapCol(TblColRef col) {
 Integer i = colMapping.get(col);
 return i == null ? -1 : i;
@@ -143,7 +154,8 @@ public class GTUtil {
 // In case of NOT(unEvaluatableFilter), we should immediately 
replace it as TRUE,
 // 

[33/50] [abbrv] kylin git commit: minor, special treatment for OLAPContext.allColumns when select star

[liyang]

minor, special treatment for OLAPContext.allColumns when select star


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3200d6a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3200d6a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3200d6a1

Branch: refs/heads/master
Commit: 3200d6a148d4368801e415f6ae969bbab7c7345e
Parents: 28a07eb
Author: Hongbin Ma 
Authored: Sun Aug 27 15:41:22 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Aug 28 10:42:57 2017 +0800

--
 .../org/apache/kylin/query/relnode/OLAPTableScan.java  | 13 +++--
 1 file changed, 11 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3200d6a1/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 7801891..dd2160b 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -210,7 +210,8 @@ public class OLAPTableScan extends TableScan implements 
OLAPRel, EnumerableRel {
 Preconditions.checkState(columnRowType == null, "OLAPTableScan MUST 
NOT be shared by more than one prent");
 
 // create context in case of non-join
-if (implementor.getContext() == null || !(implementor.getParentNode() 
instanceof OLAPJoinRel) || implementor.isNewOLAPContextRequired()) {
+if (implementor.getContext() == null || !(implementor.getParentNode() 
instanceof OLAPJoinRel)
+|| implementor.isNewOLAPContextRequired()) {
 implementor.allocateContext();
 }
 
@@ -227,6 +228,13 @@ public class OLAPTableScan extends TableScan implements 
OLAPRel, EnumerableRel {
 if (context.firstTableScan == null) {
 context.firstTableScan = this;
 }
+
+if (implementor.getParentNode() instanceof OLAPToEnumerableConverter) {
+// OLAPToEnumerableConverter on top of table scan, should be a 
select * from table
+for (TblColRef tblColRef : columnRowType.getAllColumns()) {
+context.allColumns.add(tblColRef);
+}
+}
 }
 
 public String getAlias() {
@@ -288,7 +296,8 @@ public class OLAPTableScan extends TableScan implements 
OLAPRel, EnumerableRel {
 String execFunction = genExecFunc();
 
 PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), 
this.rowType, pref.preferArray());
-MethodCallExpression exprCall = 
Expressions.call(table.getExpression(OLAPTable.class), execFunction, 
implementor.getRootExpression(), Expressions.constant(context.id));
+MethodCallExpression exprCall = 
Expressions.call(table.getExpression(OLAPTable.class), execFunction,
+implementor.getRootExpression(), 
Expressions.constant(context.id));
 return implementor.result(physType, Blocks.toBlock(exprCall));
 }
 

[36/50] [abbrv] kylin git commit: KYLIN-2816 wrap columns with backtick

[liyang]

KYLIN-2816 wrap columns with backtick


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74c904e4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74c904e4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74c904e4

Branch: refs/heads/master
Commit: 74c904e4eedc224b07c9b6463ab0b63faddd4a21
Parents: b778f33
Author: tttMelody <245915...@qq.com>
Authored: Mon Aug 28 14:19:36 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Aug 28 15:25:27 2017 +0800

--
 .../org/apache/kylin/job/JoinedFlatTable.java   | 77 ++--
 .../kylin/job/TestWrapColumnWithBacktick.java   | 57 +++
 .../metadata/model/tool/CalciteParser.java  |  8 +-
 3 files changed, 135 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/74c904e4/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 824c693..4ac5a3a 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -21,11 +21,17 @@ package org.apache.kylin.job;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.util.SqlBasicVisitor;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.metadata.model.DataModelDesc;
@@ -35,6 +41,9 @@ import org.apache.kylin.metadata.model.JoinTableDesc;
 import org.apache.kylin.metadata.model.PartitionDesc;
 import org.apache.kylin.metadata.model.TableRef;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  *
@@ -43,6 +52,7 @@ import org.apache.kylin.metadata.model.TblColRef;
 public class JoinedFlatTable {
 
 public static final String BACKTICK = "`";
+private static final Logger logger = 
LoggerFactory.getLogger(JoinedFlatTable.class);
 
 public static String getTableDir(IJoinedFlatTableDesc flatDesc, String 
storageDfsDir) {
 return storageDfsDir + "/" + flatDesc.getTableName();
@@ -130,11 +140,7 @@ public class JoinedFlatTable {
 sql.append(",");
 }
 String colTotalName = String.format("%s.%s", 
col.getTableRef().getTableName(), col.getName());
-String expressionInSourceDB = col.getExpressionInSourceDB();
-if (expressionInSourceDB.contains(".")) {
-// surround column name with back-tick, to support unicode 
column name
-expressionInSourceDB = expressionInSourceDB.replace(".", "." + 
BACKTICK) + BACKTICK;
-}
+String expressionInSourceDB = 
quoteOriginalColumnWithBacktick(col.getExpressionInSourceDB());
 if (skipAsList.contains(colTotalName)) {
 sql.append(expressionInSourceDB + sep);
 } else {
@@ -277,4 +283,65 @@ public class JoinedFlatTable {
 return sql.toString();
 }
 
+/**
+ * quote column name with back-tick, to support unicode column name
+ *
+ * @param sourceSQL
+ * @return
+ */
+static String quoteOriginalColumnWithBacktick(String sourceSQL) {
+StringBuilder result = new StringBuilder(sourceSQL);
+try {
+List> replacePoses = new ArrayList<>();
+for (SqlIdentifier id : 
SqlIdentifierVisitor.getAllSqlIdentifier(CalciteParser.getExpNode(sourceSQL))) {
+
replacePoses.add(CalciteParser.getReplacePos(id.getComponentParserPosition(1), 
"select " + sourceSQL + " from t"));
+}
+
+// latter replace position in the front of the list.
+Collections.sort(replacePoses, new Comparator>() {
+@Override
+public int compare(Pair o1, Pair o2) {
+return -(o1.getSecond() - o2.getSecond());
+}
+});
+
+// cuz the pos is add "select "'s size, so minus 7 offset.
+final int OFFSET = 7;
+for (Pair replacePos : replacePoses) {
+result.insert(replacePos.getSecond() 

[01/50] [abbrv] kylin git commit: KYLIN-2785 Handle Maprfs schema working directory

[liyang]

Repository: kylin
Updated Branches:
  refs/heads/master 727920b4d -> 363f858b8


KYLIN-2785 Handle Maprfs schema working directory


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5008866f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5008866f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5008866f

Branch: refs/heads/master
Commit: 5008866f41efd13ce5abaf8cc19c09919fe698dd
Parents: 0153263
Author: Roger Shi 
Authored: Thu Aug 10 21:34:03 2017 +0800
Committer: Roger Shi 
Committed: Sun Aug 13 22:39:54 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5008866f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3a06571..78abf22 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -220,6 +220,8 @@ abstract public class KylinConfigBase implements 
Serializable {
 cachedHdfsWorkingDirectory = root;
 if (cachedHdfsWorkingDirectory.startsWith("file:")) {
 cachedHdfsWorkingDirectory = 
cachedHdfsWorkingDirectory.replace("file:", "file://");
+} else if (cachedHdfsWorkingDirectory.startsWith("maprfs:")) {
+cachedHdfsWorkingDirectory = 
cachedHdfsWorkingDirectory.replace("maprfs:", "maprfs://");
 }
 return cachedHdfsWorkingDirectory;
 }

[18/50] [abbrv] kylin git commit: minor fix CI

[liyang]

minor fix CI


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46cd8813
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46cd8813
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46cd8813

Branch: refs/heads/master
Commit: 46cd8813902881912aaed9969151092b48f49d57
Parents: 909628f
Author: Hongbin Ma 
Authored: Tue Aug 22 11:16:41 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Aug 22 11:41:31 2017 +0800

--
 .../query/sql_computedcolumn/query04.sql| 28 +++-
 .../sql_computedcolumn/query04.sql.compare  | 28 ++--
 2 files changed, 53 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/46cd8813/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql 
b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
index 780de5d..75af19b 100644
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
+++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
@@ -1 +1,27 @@
-select count(*) as a , sum (F.DEAL_AMOUNT) as b from test_kylin_fact f inner 
join test_account a on f.seller_id = a.account_id  inner join test_country c on 
country = account_country group by F.SELLER_ID_AND_COUNTRY_NAME
+select count(*) as a , sum (TEST_KYLIN_FACT.DEAL_AMOUNT) as b , 
TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME  as c 
+
+
+
+
+FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT 
+INNER JOIN TEST_ORDER as TEST_ORDER
+ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
+ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
+ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND 
TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
+ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
+ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
+
+
+group by TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME
+
+order by b
+
+
+
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/46cd8813/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare
--
diff --git 
a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare 
b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare
index 5ee361d..66ffc07 100644
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare
+++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare
@@ -1,2 +1,26 @@
-select count(*) as a, sum (F.PRICE * F.ITEM_COUNT) as b from test_kylin_fact f
-inner join test_account a on f.seller_id = a.account_id  
inner join test_country c on country = account_country group by 
CONCAT(A.ACCOUNT_ID, C.NAME)
+select count(*) as a, sum (TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) 
as b, CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME) as c 
+
+
+
+FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT 
+INNER JOIN TEST_ORDER as TEST_ORDER
+ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
+ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
+ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND 
TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
+ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
+ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
+
+
+
+
+
+
+
+group by CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, 
SELLER_COUNTRY.NAME)
+order by b

[43/50] [abbrv] kylin git commit: KYLIN-2773 Test for join casting

[liyang]

KYLIN-2773 Test for join casting


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1eb3617c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1eb3617c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1eb3617c

Branch: refs/heads/master
Commit: 1eb3617cf1cf50da16d5f86b4f922b3e8925de4b
Parents: 87f1732
Author: Roger Shi 
Authored: Tue Aug 29 19:22:18 2017 +0800
Committer: Roger Shi 
Committed: Wed Aug 30 16:02:26 2017 +0800

--
 .../table/DEFAULT.TEST_KYLIN_FACT.json  |  2 +-
 .../apache/kylin/query/ITKylinQueryTest.java|  5 +
 .../test/resources/query/sql_join/query_00.sql  | 21 
 3 files changed, 27 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb3617c/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
--
diff --git 
a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json 
b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index e82d535..4be05f2 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -44,7 +44,7 @@
   }, {
 "id" : "8",
 "name" : "SELLER_ID",
-"datatype" : "bigint",
+"datatype" : "int",
 "data_gen" : "RAND||1000|10001000",
 "index": "T"
   }, {

http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb3617c/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
--
diff --git 
a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java 
b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 4ea4497..e332cda 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -234,6 +234,11 @@ public class ITKylinQueryTest extends KylinTestBase {
 }
 
 @Test
+public void testJoinCastQuery() throws Exception {
+execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_join", null, true);
+}
+
+@Test
 public void testCachedQuery() throws Exception {
 execAndCompQuery(getQueryFolderPrefix() + 
"src/test/resources/query/sql_cache", null, true);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1eb3617c/kylin-it/src/test/resources/query/sql_join/query_00.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_join/query_00.sql 
b/kylin-it/src/test/resources/query/sql_join/query_00.sql
new file mode 100644
index 000..024e45d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_join/query_00.sql
@@ -0,0 +1,21 @@
+-- join condition cast type
+select sum(ITEM_COUNT) as ITEM_CNT
+FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
+INNER JOIN TEST_ORDER as TEST_ORDER
+ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
+ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
+ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN EDW.TEST_CAL_DT as TEST_CAL_DT
+ON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT
+INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND 
TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN EDW.TEST_SITES as TEST_SITES
+ON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID
+INNER JOIN EDW.TEST_SELLER_TYPE_DIM as TEST_SELLER_TYPE_DIM
+ON TEST_KYLIN_FACT.SLR_SEGMENT_CD = TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD
+INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
+ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
+ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
\ No newline at end of file

[35/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as default metric framework modify comment code

[liyang]

KYLIN-2776 Using dropwizard as default metric framework
modify comment code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b778f33a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b778f33a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b778f33a

Branch: refs/heads/master
Commit: b778f33af3718fd1bf5f705b81104f424803fe78
Parents: fad75c9
Author: yiming.xu <100650...@qq.com>
Authored: Mon Aug 28 12:57:10 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Aug 28 13:50:36 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 ++
 .../kylin/common/metrics/common/Metrics.java| 40 ---
 .../common/metrics/common/MetricsConstant.java  |  2 +-
 .../common/metrics/common/MetricsFactory.java   |  2 +-
 .../metrics/common/MetricsNameBuilder.java  | 38 ++
 .../common/metrics/common/MetricsScope.java |  8 +--
 .../kylin/common/metrics/common/Metricss.java   | 40 ---
 .../metrics/metrics2/CodahaleMetrics.java   | 20 
 .../metrics2/ConsoleMetricsReporter.java|  2 +-
 .../metrics/metrics2/JmxMetricsReporter.java|  2 +-
 .../metrics2/MetricVariableRatioGauge.java  |  2 +-
 .../metrics/metrics2/Metrics2Reporter.java  |  2 +-
 .../metrics/metrics2/MetricsReporting.java  |  3 --
 .../common/metrics/perflog/PerfLogger.java  |  3 +-
 .../kylin/rest/metrics/QueryMetrics2Facade.java | 20 
 .../kylin/rest/metrics/QueryMetrics2Test.java   | 52 ++--
 16 files changed, 122 insertions(+), 118 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b778f33a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 99b9bb2..bca00e7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1149,6 +1149,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
Boolean.parseBoolean(getOptional("kylin.server.query-metrics-enabled", 
"false"));
 }
 
+public boolean getQueryMetrics2Enabled() {
+return 
Boolean.parseBoolean(getOptional("kylin.server.query-metrics2-enabled", 
"false"));
+}
+
 public int[] getQueryMetricsPercentilesIntervals() {
 String[] dft = { "60", "300", "3600" };
 return 
getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft);

http://git-wip-us.apache.org/repos/asf/kylin/blob/b778f33a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java 
b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
index dcda3cd..9bf8e7a 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
@@ -17,13 +17,13 @@
 */
 package org.apache.kylin.common.metrics.common;
 
+import java.util.concurrent.TimeUnit;
+
 import com.codahale.metrics.Counter;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Timer;
 
-import java.util.concurrent.TimeUnit;
-
 /**
  * Generic Metics interface.
  */
@@ -35,7 +35,6 @@ public interface Metrics {
 public void close() throws Exception;
 
 /**
- *
  * @param name starts a scope of a given name.  Scopes is stored as 
thread-local variable.
  */
 public void startStoredScope(String name);
@@ -43,12 +42,14 @@ public interface Metrics {
 /**
  * Closes the stored scope of a given name.
  * Note that this must be called on the same thread as where the scope was 
started.
+ *
  * @param name
  */
 public void endStoredScope(String name);
 
 /**
  * Create scope with given name and returns it.
+ *
  * @param name
  * @return
  */
@@ -56,6 +57,7 @@ public interface Metrics {
 
 /**
  * Close the given scope.
+ *
  * @param scope
  */
 public void endScope(MetricsScope scope);
@@ -64,6 +66,7 @@ public interface Metrics {
 
 /**
  * Increments a counter of the given name by 1.
+ *
  * @param name
  * @return
  */
@@ -71,6 +74,7 @@ public interface Metrics {
 
 /**
  * Increments a counter of the given name by "increment"
+ *
  * @param name
  * @param increment
  * @return
@@ -79,6 +83,7 @@ 

[48/50] [abbrv] kylin git commit: Merge commit '727920b4d3642aaa3657d90b7f3dce7dcdd68fe2'

[liyang]

Merge commit '727920b4d3642aaa3657d90b7f3dce7dcdd68fe2'


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b50030da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b50030da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b50030da

Branch: refs/heads/master
Commit: b50030da4cd64d424a82e9d2b752c00917e8e681
Parents: 068baf6 727920b
Author: Hongbin Ma 
Authored: Thu Aug 31 15:42:16 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Aug 31 21:33:14 2017 +0800

--
 assembly/pom.xml|   2 +-
 atopcalcite/pom.xml |   2 +-
 core-common/pom.xml |   2 +-
 .../org/apache/kylin/common/KylinConfig.java|  57 ++-
 .../apache/kylin/common/KylinConfigBase.java|   1 +
 .../org/apache/kylin/common/KylinConfigExt.java |   2 +-
 .../org/apache/kylin/common/KylinVersion.java   |   4 +-
 .../org/apache/kylin/common/util/ClassUtil.java |  13 +-
 core-cube/pom.xml   |   2 +-
 .../kylin/cube/gridtable/CubeCodeSystem.java|   4 +
 .../org/apache/kylin/gridtable/GTRecord.java|   8 +
 .../kylin/gridtable/GTSampleCodeSystem.java |   4 +
 .../apache/kylin/gridtable/IGTCodeSystem.java   |   3 +
 core-dictionary/pom.xml |   2 +-
 core-job/pom.xml|   2 +-
 .../job/execution/DefaultChainedExecutable.java |  13 +-
 .../kylin/job/execution/ExecutableManager.java  |  12 +-
 core-metadata/pom.xml   |   2 +-
 .../measure/bitmap/BitmapCounterFactory.java|   2 +
 .../kylin/measure/bitmap/BitmapSerializer.java  |  38 +-
 .../measure/bitmap/RoaringBitmapCounter.java|  10 +
 .../bitmap/RoaringBitmapCounterFactory.java |   5 +
 .../measure/percentile/PercentileCounter.java   |  28 +-
 .../metadata/datatype/DataTypeSerializer.java   |  13 +
 .../percentile/PercentileCounterTest.java   |  46 ---
 core-storage/pom.xml|   2 +-
 .../gtrecord/GTCubeStorageQueryBase.java|  53 ++-
 .../gtrecord/SequentialCubeTupleIterator.java   |  17 +-
 engine-mr/pom.xml   |   2 +-
 .../engine/mr/common/AbstractHadoopJob.java | 128 +++---
 .../kylin/engine/mr/common/BatchConstants.java  |   1 +
 .../kylin/engine/mr/common/HadoopCmdOutput.java |  19 +-
 .../engine/mr/common/JobInfoConverter.java  |   7 +-
 .../engine/mr/common/JobRelatedMetaUtil.java|  72 
 engine-spark/pom.xml|   2 +-
 .../engine/spark/KylinKryoRegistrator.java  |  11 +-
 .../spark/SparkBatchCubingJobBuilder2.java  |   5 +
 .../kylin/engine/spark/SparkCubingByLayer.java  | 392 +++
 .../kylin/engine/spark/SparkExecutable.java |  58 ++-
 .../spark/util/PercentileCounterSerializer.java |  55 +++
 .../spark/util/PercentileSerializerTest.java|  67 
 .../acl/0928468a-9fab-4185-9a14-6f2e7c74823f|  27 ++
 .../acl/2fbca32a-a33e-4b69-83dd-0bb8b1f8c53b|  27 ++
 .../acl/2fbca32a-a33e-4b69-83dd-0bb8b1f8c91b|  24 ++
 jdbc/pom.xml|   2 +-
 kylin-it/pom.xml|  18 +-
 .../job/ITDistributedSchedulerBaseTest.java |   2 +
 .../jdbc/ITJdbcSourceTableLoaderTest.java   | 114 ++
 .../source/jdbc/ITJdbcTableReaderTest.java  | 109 ++
 .../test/resources/query/sql_limit/query05.sql  |  21 +
 pom.xml |   2 +-
 query/pom.xml   |   2 +-
 .../query/adhoc/PushDownRunnerJdbcImpl.java |   2 -
 .../kylin/query/relnode/OLAPAggregateRel.java   |   4 +-
 .../kylin/query/relnode/OLAPWindowRel.java  |   2 -
 .../kylin/query/routing/RealizationChooser.java |   7 +-
 server-base/pom.xml |   2 +-
 .../kylin/rest/controller/CubeController.java   |  49 ++-
 .../rest/controller2/CubeControllerV2.java  |  19 +-
 .../apache/kylin/rest/service/AclService.java   |   4 +-
 .../rest/service/AclTableMigrationTool.java |  13 +-
 .../apache/kylin/rest/service/JobService.java   |   4 +
 .../apache/kylin/rest/service/QueryService.java |   4 +-
 .../kylin/rest/util/Log4jConfigListener.java|   1 +
 server/pom.xml  |   2 +-
 server/src/main/resources/kylinSecurity.xml |   2 +
 .../kylin/rest/service/JobServiceTest.java  |  33 ++
 source-hive/pom.xml |   7 +-
 .../HiveColumnCardinalityUpdateJob.java |   3 +-
 .../apache/kylin/source/jdbc/JdbcExplorer.java  |   6 +-
 source-kafka/pom.xml|   2 +-
 .../kylin/source/kafka/DateTimeParser.java  |  62 +++
 storage-hbase/pom.xml   |   2 +-
 .../kylin/storage/hbase/HBaseConnection.java|   2 +-
 

[20/50] [abbrv] kylin git commit: KYLIN-2801 fix UT

[liyang]

KYLIN-2801 fix UT


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68ae79a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68ae79a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68ae79a6

Branch: refs/heads/master
Commit: 68ae79a61c03763c0e21f27fcb79b52db849d306
Parents: a3d42f1
Author: Hongbin Ma 
Authored: Tue Aug 22 15:32:01 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Aug 22 16:00:14 2017 +0800

--
 .../DefaultPartitionConditionBuilderTest.java   | 33 +++-
 .../kylin/engine/mr/SortedColumnReaderTest.java | 17 --
 2 files changed, 41 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/68ae79a6/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
--
diff --git 
a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
 
b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
index 1b610ca..907e0e5 100644
--- 
a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
+++ 
b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
@@ -19,11 +19,24 @@
 package org.apache.kylin.metadata.model;
 
 import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-public class DefaultPartitionConditionBuilderTest {
+public class DefaultPartitionConditionBuilderTest extends 
LocalFileMetadataTestCase {
+
+@Before
+public void setup() throws Exception {
+this.createTestMetadata();
+}
+
+@After
+public void after() throws Exception {
+this.cleanupTestMetadata();
+}
+
 private PartitionDesc.DefaultPartitionConditionBuilder 
partitionConditionBuilder;
 
 @Before
@@ -38,8 +51,10 @@ public class DefaultPartitionConditionBuilderTest {
 partitionDesc.setPartitionDateColumnRef(col);
 partitionDesc.setPartitionDateColumn(col.getCanonicalName());
 partitionDesc.setPartitionDateFormat("-MM-dd");
-String condition = 
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, 
DateFormat.stringToMillis("2016-02-22"), 
DateFormat.stringToMillis("2016-02-23"));
-Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND 
UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition);
+String condition = 
partitionConditionBuilder.buildDateRangeCondition(partitionDesc,
+DateFormat.stringToMillis("2016-02-22"), 
DateFormat.stringToMillis("2016-02-23"));
+Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND 
UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'",
+condition);
 }
 
 @Test
@@ -49,7 +64,8 @@ public class DefaultPartitionConditionBuilderTest {
 partitionDesc.setPartitionTimeColumnRef(col);
 partitionDesc.setPartitionTimeColumn(col.getCanonicalName());
 partitionDesc.setPartitionTimeFormat("HH");
-String condition = 
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, 
DateFormat.stringToMillis("2016-02-22 00:00:00"), 
DateFormat.stringToMillis("2016-02-23 01:00:00"));
+String condition = 
partitionConditionBuilder.buildDateRangeCondition(partitionDesc,
+DateFormat.stringToMillis("2016-02-22 00:00:00"), 
DateFormat.stringToMillis("2016-02-23 01:00:00"));
 Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND 
UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition);
 }
 
@@ -64,8 +80,11 @@ public class DefaultPartitionConditionBuilderTest {
 partitionDesc.setPartitionTimeColumnRef(col2);
 partitionDesc.setPartitionTimeColumn(col2.getCanonicalName());
 partitionDesc.setPartitionTimeFormat("H");
-String condition = 
partitionConditionBuilder.buildDateRangeCondition(partitionDesc, 
DateFormat.stringToMillis("2016-02-22 00:00:00"), 
DateFormat.stringToMillis("2016-02-23 01:00:00"));
-Assert.assertEquals("((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND 
UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > 
'2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND 
UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < 
'2016-02-23'))", condition);
+String condition = 
partitionConditionBuilder.buildDateRangeCondition(partitionDesc,
+DateFormat.stringToMillis("2016-02-22 00:00:00"), 
DateFormat.stringToMillis("2016-02-23 01:00:00"));
+   

[08/50] [abbrv] kylin git commit: minor, update odbc driver version

[liyang]

minor, update odbc driver version


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/72a4f0f8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/72a4f0f8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/72a4f0f8

Branch: refs/heads/master
Commit: 72a4f0f8c0b8b4ad440bf1f0b5ce35e687edd858
Parents: 7a9f74c
Author: Yifan Zhang 
Authored: Fri Aug 18 16:16:51 2017 +0800
Committer: Dong Li 
Committed: Fri Aug 18 16:24:43 2017 +0800

--
 odbc/Installer(64bit)/Installer(64bit).isl | 2 +-
 odbc/Installer/Installer.isl   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/72a4f0f8/odbc/Installer(64bit)/Installer(64bit).isl
--
diff --git a/odbc/Installer(64bit)/Installer(64bit).isl 
b/odbc/Installer(64bit)/Installer(64bit).isl
index 3393ac5..a3c96d5 100644
--- a/odbc/Installer(64bit)/Installer(64bit).isl
+++ b/odbc/Installer(64bit)/Installer(64bit).isl
@@ -4395,7 +4395,7 @@ UwBpAG4AZwBsAGUASQBtAGEAZwBlAAEARQB4AHAAcgBlAHMAcwA=

PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS##IDS_PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS##

ProductCode{C8B1B296-2F8B-423F-97D5-429C9B6D2AE6}
ProductNameKylinODBCDriver 
(x64)
-   ProductVersion1.5.
+   ProductVersion2.1.0
ProgressType0install
ProgressType1Installing
ProgressType2installed

http://git-wip-us.apache.org/repos/asf/kylin/blob/72a4f0f8/odbc/Installer/Installer.isl
--
diff --git a/odbc/Installer/Installer.isl b/odbc/Installer/Installer.isl
index a23656a..f6a82a6 100644
--- a/odbc/Installer/Installer.isl
+++ b/odbc/Installer/Installer.isl
@@ -4376,7 +4376,7 @@ UwBpAG4AZwBsAGUASQBtAGEAZwBlAAEARQB4AHAAcgBlAHMAcwA=

PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS##IDS_PROGMSG_IIS_ROLLBACKWEBSERVICEEXTENSIONS##

ProductCode{A920FB5E-591B-4537-901D-7D030304}
ProductNameKylinODBCDriver 
(x86)
-   ProductVersion1.5.
+   ProductVersion2.1.0
ProgressType0install
ProgressType1Installing
ProgressType2installed

[31/50] [abbrv] kylin git commit: KYLIN-2812, Fix the wrong destination database during load kafka topic (#2074)

[liyang]

KYLIN-2812, Fix the wrong destination database during load kafka topic (#2074)



Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2e0102c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2e0102c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2e0102c

Branch: refs/heads/master
Commit: e2e0102c4bac289c1e0d61226794030fd93cb145
Parents: ff4f634
Author: Billy(Yiming) Liu 
Authored: Sun Aug 27 18:51:50 2017 +0800
Committer: GitHub 
Committed: Sun Aug 27 18:51:50 2017 +0800

--
 .../rest/controller2/StreamingControllerV2.java |  3 +-
 .../controller/StreamingControllerTest.java | 52 
 2 files changed, 53 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e0102c/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
index 54733ea..90f2a45 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller2/StreamingControllerV2.java
@@ -247,10 +247,9 @@ public class StreamingControllerV2 extends BasicController 
{
 }
 
 if (null != desc) {
-String[] dbTable = HadoopUtil.parseHiveTableName(desc.getName());
+String[] dbTable = 
HadoopUtil.parseHiveTableName(desc.getIdentity());
 desc.setName(dbTable[1]);
 desc.setDatabase(dbTable[0]);
-desc.getIdentity();
 }
 return desc;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/e2e0102c/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java
--
diff --git 
a/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java
 
b/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java
new file mode 100644
index 000..171d22e
--- /dev/null
+++ 
b/server-base/src/test/java/org/apache/kylin/rest/controller/StreamingControllerTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class StreamingControllerTest {
+
+@Test
+public void testReadTableDesc() throws IOException {
+String requestTableData = 
"{\"name\":\"my_table_name\",\"source_type\":1,\"columns\":[{\"id\":1,\"name\":"
 +
+
"\"amount\",\"datatype\":\"decimal\"},{\"id\":2,\"name\":\"category\",\"datatype\":\"varchar(256)\"},"
 +
+
"{\"id\":3,\"name\":\"order_time\",\"datatype\":\"timestamp\"},{\"id\":4,\"name\":\"device\","
 +
+
"\"datatype\":\"varchar(256)\"},{\"id\":5,\"name\":\"qty\",\"datatype\":\"int\"},{\"id\":6,\"name\":"
 +
+
"\"user_id\",\"datatype\":\"varchar(256)\"},{\"id\":7,\"name\":\"user_age\",\"datatype\":\"int\"},"
 +
+
"{\"id\":8,\"name\":\"user_gender\",\"datatype\":\"varchar(256)\"},{\"id\":9,\"name\":\"currency\","
 +
+
"\"datatype\":\"varchar(256)\"},{\"id\":10,\"name\":\"country\",\"datatype\":\"varchar(256)\"},"
 +
+
"{\"id\":11,\"name\":\"year_start\",\"datatype\":\"date\"},{\"id\":12,\"name\":\"quarter_start\","
 +
+
"\"datatype\":\"date\"},{\"id\":13,\"name\":\"month_start\",\"datatype\":\"date\"},{\"id\":14,"
 +
+
"\"name\":\"week_start\",\"datatype\":\"date\"},{\"id\":15,\"name\":\"day_start\",\"datatype\":"
 +
+ 

[05/50] [abbrv] kylin git commit: KYLIN-2791 Fix bug in readLong in BytesUtil

[liyang]

KYLIN-2791 Fix bug in readLong in BytesUtil


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa7eb692
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa7eb692
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa7eb692

Branch: refs/heads/master
Commit: fa7eb692b33218fba463f7839c00663bd83892f3
Parents: c487fe4
Author: auphyroc99 <454530...@qq.com>
Authored: Wed Aug 16 13:55:40 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Aug 16 14:13:36 2017 +0800

--
 .../src/main/java/org/apache/kylin/common/util/BytesUtil.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fa7eb692/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java 
b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index 8cc559e..d41c6b4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -133,7 +133,7 @@ public class BytesUtil {
 
 public static long readLong(ByteBuffer in) {
 long integer = 0;
-int mask = 0xff;
+long mask = 0xff;
 int shift = 0;
 for (int i = 0; i < 8; i++) {
 integer |= (in.get() << shift) & mask;

[41/50] [abbrv] kylin git commit: minor change getProject() in DataModelDesc

[liyang]

minor change getProject() in DataModelDesc


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28ac7d9a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28ac7d9a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28ac7d9a

Branch: refs/heads/master
Commit: 28ac7d9aeeeb8818207a363231b1354652f9aa8d
Parents: 487a854
Author: tttMelody <245915...@qq.com>
Authored: Tue Aug 29 17:22:41 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Aug 29 17:40:33 2017 +0800

--
 .../java/org/apache/kylin/metadata/model/DataModelDesc.java  | 4 ++--
 .../org/apache/kylin/rest/controller/ModelController.java| 2 +-
 .../java/org/apache/kylin/rest/service/ModelService.java | 8 
 .../java/org/apache/kylin/rest/service/ModelServiceTest.java | 8 
 4 files changed, 11 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28ac7d9a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index a5da833..b27d644 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -854,11 +854,11 @@ public class DataModelDesc extends RootPersistentEntity {
 }
 
 public String getProject() {
-return 
ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProjectOfModel(this.getName()).getName();
+return 
ProjectManager.getInstance(getConfig()).getProjectOfModel(this.getName()).getName();
 }
 
 public ProjectInstance getProjectInstance() {
-return 
ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProjectOfModel(this.getName());
+return 
ProjectManager.getInstance(getConfig()).getProjectOfModel(this.getName());
 }
 
 public static DataModelDesc getCopyOf(DataModelDesc orig) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/28ac7d9a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
index 4226e87..c5bee9f 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
@@ -131,7 +131,7 @@ public class ModelController extends BasicController {
 return modelRequest;
 }
 try {
-modelDesc = modelService.updateModelAndDesc(modelDesc);
+modelDesc = 
modelService.updateModelAndDesc(modelRequest.getProject(), modelDesc);
 } catch (AccessDeniedException accessDeniedException) {
 throw new ForbiddenException("You don't have right to update this 
model.");
 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/28ac7d9a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 82632eb..42cf085 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -143,8 +143,8 @@ public class ModelService extends BasicService {
 return createdDesc;
 }
 
-public DataModelDesc updateModelAndDesc(DataModelDesc desc) throws 
IOException {
-aclEvaluate.hasProjectWritePermission(desc.getProjectInstance());
+public DataModelDesc updateModelAndDesc(String project, DataModelDesc 
desc) throws IOException {
+aclEvaluate.checkProjectWritePermission(project);
 getMetadataManager().updateDataModelDesc(desc);
 return desc;
 }
@@ -428,7 +428,7 @@ public class ModelService extends BasicService {
 }
 
 public DataModelDesc updateModelToResourceStore(DataModelDesc modelDesc, 
String projectName) throws IOException {
-
aclEvaluate.hasProjectWritePermission(getProjectManager().getProject(projectName));
+aclEvaluate.checkProjectWritePermission(projectName);
 Message msg = MsgPicker.getMsg();
 
 modelDesc.setDraft(false);
@@ -445,7 +445,7 @@ public class ModelService extends BasicService {

[10/50] [abbrv] kylin git commit: KYLIN-2649 configuration of force limit on "select star" query

[liyang]

KYLIN-2649 configuration of force limit on "select star" query


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c4f125c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c4f125c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c4f125c4

Branch: refs/heads/master
Commit: c4f125c4750ef8d0c754d7a00c2ad100c6999cf6
Parents: 6ec099b
Author: Roger Shi 
Authored: Sun Aug 20 13:41:16 2017 +0800
Committer: Roger Shi 
Committed: Sun Aug 20 14:06:28 2017 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java| 6 ++
 .../src/main/java/org/apache/kylin/query/util/QueryUtil.java  | 7 +++
 2 files changed, 13 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c4f125c4/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 78abf22..df74f5e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -936,6 +936,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", 
"1"));
 }
 
+// Select star on large table is too slow for BI, add limit by default if 
missing
+// https://issues.apache.org/jira/browse/KYLIN-2649
+public int getForceLimit() {
+return Integer.parseInt(getOptional("kylin.query.force-limit", "-1"));
+}
+
 @Deprecated
 public int getScanThreshold() {
 return Integer.parseInt(getOptional("kylin.query.scan-threshold", 
"1000"));

http://git-wip-us.apache.org/repos/asf/kylin/blob/c4f125c4/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java 
b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 29e2c26..4befd33 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -44,6 +44,7 @@ public class QueryUtil {
 public static String massageSql(String sql, String project, int limit, int 
offset) {
 sql = sql.trim();
 sql = sql.replace("\r", " ").replace("\n", 
System.getProperty("line.separator"));
+KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
 
 while (sql.endsWith(";"))
 sql = sql.substring(0, sql.length() - 1);
@@ -56,6 +57,12 @@ public class QueryUtil {
 sql += ("\nOFFSET " + offset);
 }
 
+// https://issues.apache.org/jira/browse/KYLIN-2649
+if (kylinConfig.getForceLimit() > 0 &&
+!sql.toLowerCase().contains("limit") && 
sql.toLowerCase().contains("*")) {
+sql += ("\nLIMIT " + kylinConfig.getForceLimit());
+}
+
 // customizable SQL transformation
 if (queryTransformers == null) {
 initQueryTransformers();

[30/50] [abbrv] kylin git commit: KYLIN-2808, fix table not fould exception

[liyang]

KYLIN-2808, fix table not fould exception


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ff4f634f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ff4f634f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ff4f634f

Branch: refs/heads/master
Commit: ff4f634fe5ad5f96b1506a2ab368eb701f62c5dc
Parents: 1fff421
Author: Cheng Wang 
Authored: Fri Aug 25 19:10:19 2017 +0800
Committer: 成 
Committed: Sat Aug 26 00:19:08 2017 +0800

--
 .../apache/kylin/source/hive/HiveMRInput.java   | 41 +---
 1 file changed, 26 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ff4f634f/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
--
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 71a7152..096134c 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -62,10 +62,12 @@ public class HiveMRInput implements IMRInput {
 
 @SuppressWarnings("unused")
 private static final Logger logger = 
LoggerFactory.getLogger(HiveMRInput.class);
-
+
 public static String getTableNameForHCat(TableDesc table) {
 String tableName = (table.isView()) ? table.getMaterializedName() : 
table.getName();
-return String.format("%s.%s", table.getDatabase(), 
tableName).toUpperCase();
+String database = (table.isView()) ? 
KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable()
+: table.getDatabase();
+return String.format("%s.%s", database, tableName).toUpperCase();
 }
 
 @Override
@@ -139,17 +141,18 @@ public class HiveMRInput implements IMRInput {
 @Override
 public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable 
jobFlow) {
 final String cubeName = 
CubingExecutableUtil.getCubeName(jobFlow.getParams());
-final KylinConfig cubeConfig = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName).getConfig();
+final KylinConfig cubeConfig = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName)
+.getConfig();
 final String hiveInitStatements = 
JoinedFlatTable.generateHiveInitStatements(flatTableDatabase);
 
 // create flat table first
 addStepPhase1_DoCreateFlatTable(jobFlow);
-
+
 // then count and redistribute
 if (cubeConfig.isHiveRedistributeEnabled()) {
 
jobFlow.addTask(createRedistributeFlatHiveTableStep(hiveInitStatements, 
cubeName));
 }
-
+
 // special for hive
 addStepPhase1_DoMaterializeLookupTable(jobFlow);
 }
@@ -158,14 +161,14 @@ public class HiveMRInput implements IMRInput {
 final String cubeName = 
CubingExecutableUtil.getCubeName(jobFlow.getParams());
 final String hiveInitStatements = 
JoinedFlatTable.generateHiveInitStatements(flatTableDatabase);
 final String jobWorkingDir = getJobWorkingDir(jobFlow);
-
+
 jobFlow.addTask(createFlatHiveTableStep(hiveInitStatements, 
jobWorkingDir, cubeName));
 }
 
 protected void 
addStepPhase1_DoMaterializeLookupTable(DefaultChainedExecutable jobFlow) {
 final String hiveInitStatements = 
JoinedFlatTable.generateHiveInitStatements(flatTableDatabase);
 final String jobWorkingDir = getJobWorkingDir(jobFlow);
-
+
 AbstractExecutable task = 
createLookupHiveViewMaterializationStep(hiveInitStatements, jobWorkingDir);
 if (task != null) {
 jobFlow.addTask(task);
@@ -186,7 +189,8 @@ public class HiveMRInput implements IMRInput {
 return step;
 }
 
-private ShellExecutable createLookupHiveViewMaterializationStep(String 
hiveInitStatements, String jobWorkingDir) {
+private ShellExecutable createLookupHiveViewMaterializationStep(String 
hiveInitStatements,
+String jobWorkingDir) {
 ShellExecutable step = new ShellExecutable();
 
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP);
 
@@ -215,21 +219,25 @@ public class HiveMRInput implements IMRInput {
 if (lookUpTableDesc.isView()) {
 StringBuilder createIntermediateTableHql = new 
StringBuilder();
 createIntermediateTableHql.append("DROP TABLE IF EXISTS " 
+ intermediate + 

[03/50] [abbrv] kylin git commit: minor, fix getFromNode when SQL has limit/order by clause

[liyang]

minor, fix getFromNode  when SQL has limit/order by clause



Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2eca43c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2eca43c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2eca43c4

Branch: refs/heads/master
Commit: 2eca43c40723cce0f12a56a1299894bbf5803d03
Parents: c8bab3d
Author: Jiatao Tao <245915...@qq.com>
Authored: Mon Aug 14 05:05:31 2017 -0500
Committer: GitHub 
Committed: Mon Aug 14 05:05:31 2017 -0500

--
 .../metadata/model/tool/CalciteParser.java  | 20 ++--
 .../apache/kylin/query/util/PushDownUtil.java   |  4 
 .../kylin/query/util/PushDownUtilTest.java  | 10 +-
 3 files changed, 23 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2eca43c4/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
index a48abae..01f0b19 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
@@ -29,6 +29,7 @@ import org.apache.calcite.sql.SqlIdentifier;
 import org.apache.calcite.sql.SqlLiteral;
 import org.apache.calcite.sql.SqlNode;
 import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlOrderBy;
 import org.apache.calcite.sql.SqlSelect;
 import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.calcite.sql.parser.SqlParser;
@@ -69,14 +70,21 @@ public class CalciteParser {
 }
 
 public static SqlNode getFromNode(String sql) {
-//When the sql have limit clause, calcite will parse it as a SqlOrder 
Object.
-sql = sql.split("LIMIT")[0];
-SqlNode fromNode = null;
+SqlNode node = null;
 try {
-fromNode = ((SqlSelect) (CalciteParser.parse(sql))).getFrom();
+node = CalciteParser.parse(sql);
 } catch (SqlParseException e) {
-throw new RuntimeException("Failed to parse expression \'" + sql
-+ "\', please make sure the expression is valid");
+throw new RuntimeException(
+"Failed to parse expression \'" + sql + "\', please make 
sure the expression is valid");
+}
+//When the sql have limit clause, calcite will parse it as a SqlOrder 
Object.
+SqlNode fromNode = null;
+if (node instanceof SqlOrderBy) {
+SqlOrderBy orderBy = (SqlOrderBy) node;
+fromNode = ((SqlSelect) orderBy.query).getFrom();
+} else {
+SqlSelect sqlSelect = (SqlSelect) node;
+fromNode = sqlSelect.getFrom();
 }
 return fromNode;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2eca43c4/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java 
b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index b2248f8..51352d2 100644
--- a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -112,6 +112,10 @@ public class PushDownUtil {
 FromTablesVisitor ftv = new FromTablesVisitor();
 fromNode.accept(ftv);
 List tablesWithoutSchema = ftv.getTablesWithoutSchema();
+// sql do not need completion
+if (tablesWithoutSchema.isEmpty()) {
+return inputSql;
+}
 
 List> tablesPos = new ArrayList<>();
 for (SqlNode tables : tablesWithoutSchema) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/2eca43c4/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
--
diff --git 
a/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java 
b/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
index e2ccd9e..8ed58f7 100644
--- a/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
@@ -25,12 +25,12 @@ import org.mockito.Mockito;
 public class PushDownUtilTest {
 @Test
 public void testSchemaCompletion() {
-String sql1 = "SELECT a \n"+
+String sql1 = "SELECT a \n" +
 "FROM a.KYLIN_SALES as KYLIN_SALES\n" +
 

[17/50] [abbrv] kylin git commit: KYLIN-2575 code refactors

[liyang]

KYLIN-2575 code refactors


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aed840f9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aed840f9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aed840f9

Branch: refs/heads/master
Commit: aed840f9343e3e62a711517900c357a091e75619
Parents: 8906d13
Author: Hongbin Ma 
Authored: Mon Aug 21 11:51:15 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Aug 21 22:55:59 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  18 +-
 .../common/util/CaseInsensitiveString.java  |  63 +++
 .../kylin/common/util/CliCommandExecutor.java   |   5 +-
 .../org/apache/kylin/common/util/BasicTest.java |   8 +-
 .../cube/cuboid/DefaultCuboidScheduler.java |  18 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |  92 ++
 .../cube/model/TooManyCuboidException.java  |  25 +++
 .../org/apache/kylin/cube/CubeDescTest.java |   7 +-
 .../org/apache/kylin/job/JoinedFlatTable.java   |   2 +-
 .../apache/kylin/metadata/model/ColumnDesc.java |  15 +-
 .../metadata/model/ComputedColumnDesc.java  |  73 ++--
 .../kylin/metadata/model/DataModelDesc.java |  21 ++-
 .../metadata/model/tool/CalciteParser.java  | 102 ++-
 .../kylin/metadata/project/ProjectManager.java  |   9 +-
 .../kylin/model/tool/CalciteParserTest.java |  16 --
 .../gtrecord/GTCubeStorageQueryBase.java|  54 --
 .../localmeta/cube_desc/ci_inner_join_cube.json |  46 +++--
 .../localmeta/cube_desc/ci_left_join_cube.json  |  46 +++--
 .../cube_desc/ut_inner_join_cube_partial.json   |  28 +--
 .../model_desc/ci_inner_join_model.json |  52 --
 .../model_desc/ci_left_join_model.json  |  54 --
 .../query/sql_computedcolumn/query01.sql|   2 +-
 .../query/sql_computedcolumn/query02.sql|   2 +-
 .../query/sql_computedcolumn/query03.sql|   2 +-
 .../query/sql_verifyCount/query01.sql.expected  |   2 +-
 .../query/sql_verifyCount/query03.sql.expected  |   2 +-
 .../query/sql_verifyCount/query04.sql.expected  |   2 +-
 .../query/sql_verifyCount/query10.sql.expected  |   2 +-
 .../query/sql_verifyCount/query11.sql.expected  |   2 +-
 .../query/adhoc/PushDownRunnerJdbcImpl.java |   2 +-
 .../query/enumerator/LookupTableEnumerator.java |   2 +-
 .../kylin/query/relnode/ColumnRowType.java  |   4 +
 .../kylin/query/relnode/OLAPProjectRel.java |   2 +-
 .../kylin/query/relnode/OLAPTableScan.java  |   2 +-
 .../apache/kylin/query/schema/OLAPSchema.java   |  12 +-
 .../kylin/query/schema/OLAPSchemaFactory.java   |   2 +-
 .../apache/kylin/query/schema/OLAPTable.java|  22 ++-
 .../query/util/KeywordDefaultDirtyHack.java |   2 +-
 .../apache/kylin/query/util/PushDownUtil.java   | 174 ++-
 .../org/apache/kylin/query/util/QueryUtil.java  |  25 ++-
 .../kylin/query/util/PushDownUtilTest.java  |  43 -
 .../apache/kylin/query/util/QueryUtilTest.java  |  10 +-
 .../rest/controller2/ModelControllerV2.java |  34 +++-
 .../apache/kylin/rest/service/ModelService.java | 113 +++-
 .../apache/kylin/rest/service/QueryService.java |  75 
 45 files changed, 861 insertions(+), 433 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index df74f5e..0f5f3a9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -605,7 +605,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 public Map getKafkaConfigOverride() {
 return getPropertiesByPrefix("kylin.source.kafka.config-override.");
 }
-
+
 // 

 // SOURCE.JDBC
 // 

@@ -613,23 +613,23 @@ abstract public class KylinConfigBase 

[39/50] [abbrv] kylin git commit: Revert "KYLIN-2816 support hive unicode column name"

[liyang]

Revert "KYLIN-2816 support hive unicode column name"

This reverts commit 3712464e5e09c66b4c17cdb0c19e420c61c13358.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/487a854b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/487a854b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/487a854b

Branch: refs/heads/master
Commit: 487a854bfece7b9fc72b0118a40d51100eca8463
Parents: 81e9f57
Author: Hongbin Ma 
Authored: Mon Aug 28 19:19:24 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Aug 29 10:36:10 2017 +0800

--
 .../kylin/common/util/HiveCmdBuilder.java   |  5 ++--
 .../kylin/common/util/HiveCmdBuilderTest.java   |  2 +-
 .../org/apache/kylin/job/JoinedFlatTable.java   | 28 
 .../apache/kylin/source/hive/HiveMRInput.java   |  2 +-
 4 files changed, 9 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 191bbc0..2f6b9a0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -66,12 +66,11 @@ public class HiveCmdBuilder {
 
 switch (clientMode) {
 case CLI:
-// use single-quote to ignore the executing back-ticks surrounded 
column name in shell
-buf.append("hive -e \'");
+buf.append("hive -e \"");
 for (String statement : statements) {
 buf.append(statement).append("\n");
 }
-buf.append("\'");
+buf.append("\"");
 buf.append(parseProps());
 break;
 case BEELINE:

http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index d2938b5..d69d4d2 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -60,7 +60,7 @@ public class HiveCmdBuilderTest {
 hiveCmdBuilder.addStatement("SHOW\n TABLES;");
 hiveCmdBuilder.setHiveConfProps(hiveProps);
 hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite);
-assertEquals("hive -e \'USE default;\nDROP TABLE test;\nSHOW\n 
TABLES;\n\' --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
+assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n 
TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
 }
 
 @Test

http://git-wip-us.apache.org/repos/asf/kylin/blob/487a854b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 824c693..acb29e1 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -42,8 +42,6 @@ import org.apache.kylin.metadata.model.TblColRef;
 
 public class JoinedFlatTable {
 
-public static final String BACKTICK = "`";
-
 public static String getTableDir(IJoinedFlatTableDesc flatDesc, String 
storageDfsDir) {
 return storageDfsDir + "/" + flatDesc.getTableName();
 }
@@ -77,7 +75,7 @@ public class JoinedFlatTable {
 ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
 }
 ddl.append("STORED AS " + format + "\n");
-ddl.append("LOCATION '\"'\"'" + getTableDir(flatDesc, storageDfsDir) + 
"'\"'\"';").append("\n");
+ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + 
"';").append("\n");
 return ddl.toString();
 }
 
@@ -130,15 +128,10 @@ public class JoinedFlatTable {
 sql.append(",");
 }
 String colTotalName = String.format("%s.%s", 
col.getTableRef().getTableName(), col.getName());
-String expressionInSourceDB = col.getExpressionInSourceDB();
-if (expressionInSourceDB.contains(".")) {
-// surround column name with back-tick, to support 

[45/50] [abbrv] kylin git commit: KYLIN-2790 Add extending point for column family

[liyang]

KYLIN-2790 Add extending point for column family


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d62cb58b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d62cb58b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d62cb58b

Branch: refs/heads/master
Commit: d62cb58bde55766dbe87bbad29953dd1243c2014
Parents: 4782289
Author: auphyroc99 <454530...@qq.com>
Authored: Wed Aug 16 10:34:02 2017 +0800
Committer: Roger Shi 
Committed: Wed Aug 30 18:13:52 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 
 .../org/apache/kylin/cube/model/CubeDesc.java   | 18 ++--
 .../kylin/cube/model/HBaseMappingDesc.java  | 18 
 .../org/apache/kylin/gridtable/GTRecord.java| 22 
 4 files changed, 60 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a113327..47806a5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -286,6 +286,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
 return (DistributedLockFactory) ClassUtil.newInstance(clsName);
 }
+
+public String getHBaseMappingAdapter() {
+return getOptional("kylin.metadata.hbasemapping-adapter");
+}
 
 // 

 // DICTIONARY & SNAPSHOT

http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 0e22587..d263e1f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -22,6 +22,7 @@ import static 
com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 
+import java.lang.reflect.Method;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
@@ -603,8 +604,21 @@ public class CubeDesc extends RootPersistentEntity 
implements IEngineAware {
 validateAggregationGroups(); // check if aggregation group is valid
 validateAggregationGroupsCombination();
 
-if (hbaseMapping != null) {
-hbaseMapping.init(this);
+String hbaseMappingAdapterName = config.getHBaseMappingAdapter();
+
+if (hbaseMappingAdapterName != null) {
+try {
+Class hbaseMappingAdapterClass = 
Class.forName(hbaseMappingAdapterName);
+Method initMethod = 
hbaseMappingAdapterClass.getMethod("initHBaseMapping", CubeDesc.class);
+initMethod.invoke(null, this);
+} catch (Exception e) {
+logger.error("Wrong configuration for 
kylin.metadata.hbasemapping-adapter: class "
++ hbaseMappingAdapterName + " not found. ");
+}
+} else {
+if (hbaseMapping != null) {
+hbaseMapping.init(this);
+}
 }
 
 initMeasureReferenceToColumnFamily();

http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index d1e5829..77b4b14 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -87,6 +87,24 @@ public class HBaseMappingDesc implements 
java.io.Serializable {
 }
 }
 
+public void initAsSeparatedColumns(CubeDesc cubeDesc) {
+cubeRef = cubeDesc;
+
+int cfNum = cubeDesc.getMeasures().size();
+columnFamily = new HBaseColumnFamilyDesc[cfNum];
+
+for (int i = 0; i < cfNum; i++) {
+

[34/50] [abbrv] kylin git commit: minor, avoid buffer size overflow

[liyang]

minor, avoid buffer size overflow


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fad75c93
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fad75c93
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fad75c93

Branch: refs/heads/master
Commit: fad75c93f42d252dd1ee2ec7b03df716c15fb2f4
Parents: 3200d6a
Author: Hongbin Ma 
Authored: Mon Aug 28 12:18:23 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Aug 28 12:25:12 2017 +0800

--
 .../kylin/metadata/filter/TupleFilterSerializer.java   | 13 +
 1 file changed, 9 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/fad75c93/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index 63153ef..d434f0b 100755
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -69,8 +69,11 @@ public class TupleFilterSerializer {
 internalSerialize(rootFilter, decorator, buffer, cs);
 break;
 } catch (BufferOverflowException e) {
-logger.info("Buffer size {} cannot hold the filter, resizing 
to 4 times", bufferSize);
-bufferSize *= 4;
+if (bufferSize == (1 << 30))
+throw e;
+
+logger.info("Buffer size {} cannot hold the filter, resizing 
to 2 times", bufferSize);
+bufferSize = bufferSize << 1;
 }
 }
 byte[] result = new byte[buffer.position()];
@@ -78,7 +81,8 @@ public class TupleFilterSerializer {
 return result;
 }
 
-private static void internalSerialize(TupleFilter filter, Decorator 
decorator, ByteBuffer buffer, IFilterCodeSystem cs) {
+private static void internalSerialize(TupleFilter filter, Decorator 
decorator, ByteBuffer buffer,
+IFilterCodeSystem cs) {
 if (decorator != null) { // give decorator a chance to manipulate the 
output filter
 filter = decorator.onSerialize(filter);
 }
@@ -203,7 +207,8 @@ public class TupleFilterSerializer {
 if (extendedTupleFilters.containsKey(op)) {
 try {
 filter = (TupleFilter) 
extendedTupleFilters.get(op).getConstructor().newInstance();
-} catch (InstantiationException | IllegalAccessException | 
InvocationTargetException | NoSuchMethodException e) {
+} catch (InstantiationException | IllegalAccessException | 
InvocationTargetException
+| NoSuchMethodException e) {
 throw new RuntimeException(e);
 }
 } else {

[26/50] [abbrv] kylin git commit: minor, fix flanky test query

[liyang]

minor, fix flanky test query


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd9a5152
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd9a5152
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd9a5152

Branch: refs/heads/master
Commit: cd9a515286eb7ba0553339cfb98d86ea10377a90
Parents: 8e07884
Author: Hongbin Ma 
Authored: Wed Aug 23 14:05:47 2017 +0800
Committer: Hongbin Ma 
Committed: Thu Aug 24 10:30:24 2017 +0800

--
 kylin-it/src/test/resources/query/sql/query98.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/cd9a5152/kylin-it/src/test/resources/query/sql/query98.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query98.sql 
b/kylin-it/src/test/resources/query/sql/query98.sql
index d73a4cb..81736d9 100644
--- a/kylin-it/src/test/resources/query/sql/query98.sql
+++ b/kylin-it/src/test/resources/query/sql/query98.sql
@@ -16,6 +16,6 @@
 -- limitations under the License.
 --
 
-select cal_dt, sum(price) as x,leaf_categ_id as GMV 
+select cal_dt,leaf_categ_id as x, sum(price) as GMV 
  from test_kylin_fact 
  group by leaf_categ_id, cal_dt

[14/50] [abbrv] kylin git commit: minor, more checks on computed column

[liyang]

minor, more checks on computed column


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/788c7313
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/788c7313
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/788c7313

Branch: refs/heads/master
Commit: 788c731337cf76aab14ac388fa06782943cdf711
Parents: 6e28a02
Author: Hongbin Ma 
Authored: Mon Aug 21 22:10:36 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Aug 21 22:55:59 2017 +0800

--
 .../kylin/metadata/model/ComputedColumnDesc.java   | 17 -
 .../apache/kylin/metadata/model/DataModelDesc.java |  2 +-
 .../resources/query/sql_computedcolumn/query04.sql |  1 +
 .../query/sql_computedcolumn/query04.sql.compare   |  2 ++
 4 files changed, 20 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/788c7313/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 9911fd2..bd785c8 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -18,6 +18,7 @@
 package org.apache.kylin.metadata.model;
 
 import java.io.Serializable;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.calcite.sql.SqlAsOperator;
@@ -55,7 +56,9 @@ public class ComputedColumnDesc implements Serializable {
 @JsonProperty
 private String comment;
 
-public void init(Set aliasSet, String rootFactTableName) {
+public void init(Map aliasMap, String rootFactTableName) 
{
+Set aliasSet = aliasMap.keySet();
+
 Preconditions.checkNotNull(tableIdentity, "tableIdentity is null");
 Preconditions.checkNotNull(columnName, "columnName is null");
 Preconditions.checkNotNull(expression, "expression is null");
@@ -81,6 +84,18 @@ public class ComputedColumnDesc implements Serializable {
 throw new IllegalArgumentException("Computed column has to be 
defined on fact table");
 }
 
+for (TableRef tableRef : aliasMap.values()) {
+if (!rootFactTableName.equals(tableRef.getAlias())) {
+for (TblColRef tblColRef : tableRef.getColumns()) {
+if (this.columnName.equals(tblColRef.getName())) {
+throw new IllegalArgumentException(
+"Computed column name " + columnName + " is 
already found on table "
++ tableRef.getTableIdentity() + ", use 
a different computed column name");
+}
+}
+}
+}
+
 if ("true".equals(System.getProperty("needCheckCC"))) { //conditional 
execute this because of the calcite dependency is to available every where
 try {
 simpleParserCheck(expression, aliasSet);

http://git-wip-us.apache.org/repos/asf/kylin/blob/788c7313/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index fd609b8..a5da833 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -501,7 +501,7 @@ public class DataModelDesc extends RootPersistentEntity {
 
 for (ComputedColumnDesc newCC : this.computedColumnDescs) {
 
-newCC.init(aliasMap.keySet(), rootFactTableRef.getAlias());
+newCC.init(aliasMap, rootFactTableRef.getAlias());
 final String newCCFullName = newCC.getFullName();
 final String newCCColumnName = newCC.getColumnName();
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/788c7313/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql 
b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
new file mode 100644
index 000..780de5d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
@@ -0,0 +1 @@
+select count(*) as a , sum (F.DEAL_AMOUNT) as b from test_kylin_fact f inner 
join test_account a on f.seller_id = 

[47/50] [abbrv] kylin git commit: minor, refactor pushdown code

[liyang]

minor, refactor pushdown code


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/068baf6b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/068baf6b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/068baf6b

Branch: refs/heads/master
Commit: 068baf6b3ec13fe6e68e9a41593dcd139b4a8597
Parents: a4aeb77
Author: Li Yang 
Authored: Wed Aug 30 21:18:19 2017 +0800
Committer: Roger Shi 
Committed: Thu Aug 31 10:37:32 2017 +0800

--
 .../kylin/common/debug/BackdoorToggles.java |  19 +++-
 .../source/adhocquery/IPushDownRunner.java  |   2 +-
 .../org/apache/kylin/query/KylinTestBase.java   |  11 +-
 .../query/adhoc/PushDownRunnerJdbcImpl.java |   5 +-
 .../apache/kylin/query/relnode/OLAPContext.java |  11 +-
 .../apache/kylin/query/relnode/OLAPJoinRel.java |   1 +
 .../relnode/OLAPToEnumerableConverter.java  |  61 ++-
 .../routing/RoutingIndicatorException.java  |  35 +++
 .../apache/kylin/query/util/PushDownUtil.java   | 104 ++-
 .../apache/kylin/rest/service/QueryService.java |  37 ---
 10 files changed, 145 insertions(+), 141 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/068baf6b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 8cb48b6..88c7c16 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -40,6 +40,14 @@ public class BackdoorToggles {
 _backdoorToggles.set(toggles);
 }
 
+public static void addToggle(String key, String value) {
+Map map = _backdoorToggles.get();
+if (map == null) {
+setToggles(Maps. newHashMap());
+}
+_backdoorToggles.get().put(key, value);
+}
+
 public static void addToggles(Map toggles) {
 Map map = _backdoorToggles.get();
 if (map == null) {
@@ -47,6 +55,15 @@ public class BackdoorToggles {
 }
 _backdoorToggles.get().putAll(toggles);
 }
+
+// try avoid using this generic method
+public static String getToggle(String key) {
+Map map = _backdoorToggles.get();
+if (map == null)
+return null;
+
+return map.get(key);
+}
 
 public static String getCoprocessorBehavior() {
 return getString(DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR);
@@ -109,7 +126,7 @@ public class BackdoorToggles {
 public static boolean getPrepareOnly() {
 return getBoolean(DEBUG_TOGGLE_PREPARE_ONLY);
 }
-
+
 private static String getString(String key) {
 Map toggles = _backdoorToggles.get();
 if (toggles == null) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/068baf6b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
 
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
index 0336bfb..9983f5c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
@@ -47,6 +47,6 @@ public interface IPushDownRunner {
  *
  * @throws Exception if running pushdown fails
  */
-boolean executeUpdate(String sql) throws Exception;
+void executeUpdate(String sql) throws Exception;
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/068baf6b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java 
b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 8f143c7..d3db995 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -70,7 +70,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Lists;
 import com.google.common.io.Files;
 
 /**
@@ -261,14 +260,12 @@ public class KylinTestBase {
 
 return output(resultSet, needDisplay);
 } catch (SQLException 

[11/50] [abbrv] kylin git commit: minor, fix odbc version

[liyang]

minor, fix odbc version


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c8de93c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c8de93c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c8de93c

Branch: refs/heads/master
Commit: 1c8de93c33cdcb4b032409d0fea37885014b3cab
Parents: c4f125c
Author: lidongsjtu 
Authored: Mon Aug 21 12:55:05 2017 +0800
Committer: Yifan Zhang 
Committed: Mon Aug 21 13:48:25 2017 +0800

--
 odbc/Driver/GODBC.RC   | 480 ++--
 odbc/Driver/resource.h | 105 +-
 2 files changed, 292 insertions(+), 293 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c8de93c/odbc/Driver/GODBC.RC
--
diff --git a/odbc/Driver/GODBC.RC b/odbc/Driver/GODBC.RC
index 9e0f4c3..73b17f9 100644
--- a/odbc/Driver/GODBC.RC
+++ b/odbc/Driver/GODBC.RC
@@ -1,240 +1,240 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Microsoft Visual C++ generated resource script.
-//
-#include "resource.h"
-
-#define APSTUDIO_READONLY_SYMBOLS
-/
-//
-// Generated from the TEXTINCLUDE 2 resource.
-//
-#include "afxres.h"
-
-/
-#undef APSTUDIO_READONLY_SYMBOLS
-
-/
-// Chinese (Simplified, PRC) resources
-
-#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_CHS)
-LANGUAGE LANG_CHINESE, SUBLANG_CHINESE_SIMPLIFIED
-#pragma code_page(936)
-
-/
-//
-// Bitmap
-//
-
-IDB_BITMAP1 BITMAP  "banner.bmp"
-#endif// Chinese (Simplified, PRC) resources
-/
-
-
-/
-// English (United States) resources
-
-#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
-LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
-#pragma code_page(1252)
-
-#ifdef APSTUDIO_INVOKED
-/
-//
-// TEXTINCLUDE
-//
-
-1 TEXTINCLUDE 
-BEGIN
-"resource.h\0"
-END
-
-2 TEXTINCLUDE 
-BEGIN
-"#include ""afxres.h""\r\n"
-"\0"
-END
-
-3 TEXTINCLUDE 
-BEGIN
-"\r\n"
-"\0"
-END
-
-#endif// APSTUDIO_INVOKED
-
-
-/
-//
-// Version
-//
-
-VS_VERSION_INFO VERSIONINFO
- FILEVERSION 1,0,0,1
- PRODUCTVERSION 1,0,0,1
- FILEFLAGSMASK 0x3fL
-#ifdef _DEBUG
- FILEFLAGS 0x1L
-#else
- FILEFLAGS 0x0L
-#endif
- FILEOS 0x40004L
- FILETYPE 0x2L
- FILESUBTYPE 0x0L
-BEGIN
-BLOCK "StringFileInfo"
-BEGIN
-BLOCK "040904b0"
-BEGIN
-VALUE "CompanyName", "kylinolap"
-VALUE "FileDescription", "Kylin ODBC Driver"
-VALUE "FileVersion", "1, 0, 0, 1"
-VALUE "InternalName", "KylinODBC"
-VALUE "LegalCopyright", "Copyright ?2014"
-VALUE "OriginalFilename", "driver.dll"
-VALUE "ProductName", "Kylin ODBC Driver"
-VALUE "ProductVersion", "1, 0, 0, 1"
-END
-END
-BLOCK "VarFileInfo"
-BEGIN
-VALUE "Translation", 0x409, 1200
-END
-END
-
-
-/
-//
-// Dialog
-//
-
-IDD_DSN_CFG1 DIALOGEX 0, 0, 265, 210
-STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_POPUP | WS_CAPTION
-CAPTION "Apache Kylin Driver Connection"
-FONT 8, "MS Sans Serif", 0, 0, 0x0
-BEGIN
-PUSHBUTTON  "Cancel",IDCANCEL,137,182,50,14
-LTEXT   "Server Host",IDC_STATIC,44,74,38,8
-LTEXT   "Port",IDC_STATIC,68,90,14,8
-EDITTEXTIDC_SERVER,91,72,110,12,ES_AUTOHSCROLL
-EDITTEXT

[15/50] [abbrv] kylin git commit: KYLIN-2575 code refactor

[liyang]

KYLIN-2575 code refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e28a020
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e28a020
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e28a020

Branch: refs/heads/master
Commit: 6e28a020309796289d892a704f6a7f62cc7512c9
Parents: aed840f
Author: Hongbin Ma 
Authored: Mon Aug 21 20:35:15 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Aug 21 22:55:59 2017 +0800

--
 .../kylin/measure/MeasureTypeFactory.java   |  24 ++-
 .../metadata/model/ComputedColumnDesc.java  |  37 +++-
 .../kylin/metadata/model/DataModelDesc.java |  19 ++
 .../metadata/model/tool/CalciteParser.java  |  43 ++--
 .../adhocquery/HivePushDownConverter.java   |   2 +-
 .../source/adhocquery/IPushDownConverter.java   |   2 +-
 .../kylin/model/tool/CalciteParserTest.java |   6 +-
 .../apache/kylin/query/util/PushDownUtil.java   | 119 ++-
 .../kylin/query/util/PushDownUtilTest.java  | 207 +--
 .../apache/kylin/rest/service/ModelService.java |   3 +-
 10 files changed, 150 insertions(+), 312 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6e28a020/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index 7f3a5f1..be5e12c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -121,7 +121,8 @@ abstract public class MeasureTypeFactory {
 logger.info("Checking custom measure types from kylin 
config: " + customFactory);
 factoryInsts.add((MeasureTypeFactory) 
Class.forName(customFactory).newInstance());
 } catch (Exception e) {
-throw new IllegalArgumentException("Unrecognized 
MeasureTypeFactory classname: " + customFactory, e);
+throw new IllegalArgumentException("Unrecognized 
MeasureTypeFactory classname: " + customFactory,
+e);
 }
 }
 } catch (KylinConfigCannotInitException e) {
@@ -132,10 +133,12 @@ abstract public class MeasureTypeFactory {
 for (MeasureTypeFactory factory : factoryInsts) {
 String funcName = factory.getAggrFunctionName();
 if (funcName.equals(funcName.toUpperCase()) == false)
-throw new IllegalArgumentException("Aggregation function name 
'" + funcName + "' must be in upper case");
+throw new IllegalArgumentException(
+"Aggregation function name '" + funcName + "' must be 
in upper case");
 String dataTypeName = factory.getAggrDataTypeName();
 if (dataTypeName.equals(dataTypeName.toLowerCase()) == false)
-throw new IllegalArgumentException("Aggregation data type name 
'" + dataTypeName + "' must be in lower case");
+throw new IllegalArgumentException(
+"Aggregation data type name '" + dataTypeName + "' 
must be in lower case");
 Class> serializer = 
factory.getAggrDataTypeSerializer();
 
 logger.info("registering " + funcName + "(" + dataTypeName + "), " 
+ factory.getClass());
@@ -153,7 +156,8 @@ abstract public class MeasureTypeFactory {
 }
 
 private static void registerUDAF(MeasureTypeFactory factory) {
-MeasureType type = 
factory.createMeasureType(factory.getAggrFunctionName(), 
DataType.getType(factory.getAggrDataTypeName()));
+MeasureType type = 
factory.createMeasureType(factory.getAggrFunctionName(),
+DataType.getType(factory.getAggrDataTypeName()));
 Map> udafs = type.getRewriteCalciteAggrFunctions();
 if (type.needRewrite() == false || udafs == null)
 return;
@@ -164,7 +168,8 @@ abstract public class MeasureTypeFactory {
 continue; // 

[37/50] [abbrv] kylin git commit: minor, fix some error messages.

[liyang]

minor, fix some error messages.


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/480592d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/480592d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/480592d4

Branch: refs/heads/master
Commit: 480592d49b8479d4cbba74e889eb5e256d9b2ef8
Parents: 74c904e
Author: Roger Shi 
Authored: Mon Aug 28 15:41:59 2017 +0800
Committer: Roger Shi 
Committed: Mon Aug 28 15:47:38 2017 +0800

--
 server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java | 2 +-
 server-base/src/main/java/org/apache/kylin/rest/msg/Message.java   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/480592d4/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
--
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java 
b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
index ece4b83..2b1bf8e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
@@ -187,7 +187,7 @@ public class CnMessage extends Message {
 }
 
 public String getUPDATE_MODEL_KEY_FIELD() {
-return "已使用的维度和度量与现有的连接æ 
‘不能被修改";
+return "由于维度、度量或者连接å…
³ç³»è¢«ä¿®æ”¹å¯¼è‡´ä¸Žå­˜åœ¨çš„cube定义不一致，因而当前模型无
法保存。";
 }
 
 public String getBROKEN_MODEL_DESC() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/480592d4/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
--
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java 
b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
index 8137c34..146a0a5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
@@ -187,7 +187,7 @@ public class Message {
 }
 
 public String getUPDATE_MODEL_KEY_FIELD() {
-return "Dimensions and measures in use and existing join tree cannot 
be modified.";
+return "Model cannot save because there are dimensions, measures or 
join relations modified to be inconsistent with existing cube.";
 }
 
 public String getBROKEN_MODEL_DESC() {

[32/50] [abbrv] kylin git commit: KYLIN-2816 support hive unicode column name

[liyang]

KYLIN-2816 support hive unicode column name



Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28a07ebd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28a07ebd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28a07ebd

Branch: refs/heads/master
Commit: 28a07ebd3983c325c64c0642e7dedbf32c284f39
Parents: e2e0102
Author: Billy(Yiming) Liu 
Authored: Sun Aug 27 18:55:03 2017 +0800
Committer: GitHub 
Committed: Sun Aug 27 18:55:03 2017 +0800

--
 .../kylin/common/util/HiveCmdBuilder.java   |  5 ++--
 .../kylin/common/util/HiveCmdBuilderTest.java   |  2 +-
 .../org/apache/kylin/job/JoinedFlatTable.java   | 28 
 .../apache/kylin/source/hive/HiveMRInput.java   |  2 +-
 4 files changed, 28 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java 
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 2f6b9a0..191bbc0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -66,11 +66,12 @@ public class HiveCmdBuilder {
 
 switch (clientMode) {
 case CLI:
-buf.append("hive -e \"");
+// use single-quote to ignore the executing back-ticks surrounded 
column name in shell
+buf.append("hive -e \'");
 for (String statement : statements) {
 buf.append(statement).append("\n");
 }
-buf.append("\"");
+buf.append("\'");
 buf.append(parseProps());
 break;
 case BEELINE:

http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
 
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index d69d4d2..d2938b5 100644
--- 
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++ 
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -60,7 +60,7 @@ public class HiveCmdBuilderTest {
 hiveCmdBuilder.addStatement("SHOW\n TABLES;");
 hiveCmdBuilder.setHiveConfProps(hiveProps);
 hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite);
-assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n 
TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
+assertEquals("hive -e \'USE default;\nDROP TABLE test;\nSHOW\n 
TABLES;\n\' --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
 }
 
 @Test

http://git-wip-us.apache.org/repos/asf/kylin/blob/28a07ebd/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java 
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index acb29e1..824c693 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -42,6 +42,8 @@ import org.apache.kylin.metadata.model.TblColRef;
 
 public class JoinedFlatTable {
 
+public static final String BACKTICK = "`";
+
 public static String getTableDir(IJoinedFlatTableDesc flatDesc, String 
storageDfsDir) {
 return storageDfsDir + "/" + flatDesc.getTableName();
 }
@@ -75,7 +77,7 @@ public class JoinedFlatTable {
 ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
 }
 ddl.append("STORED AS " + format + "\n");
-ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + 
"';").append("\n");
+ddl.append("LOCATION '\"'\"'" + getTableDir(flatDesc, storageDfsDir) + 
"'\"'\"';").append("\n");
 return ddl.toString();
 }
 
@@ -128,10 +130,15 @@ public class JoinedFlatTable {
 sql.append(",");
 }
 String colTotalName = String.format("%s.%s", 
col.getTableRef().getTableName(), col.getName());
+String expressionInSourceDB = col.getExpressionInSourceDB();
+if (expressionInSourceDB.contains(".")) {
+// surround column name with back-tick, to support unicode 
column name
+expressionInSourceDB = 

[09/50] [abbrv] kylin git commit: minor, fix display of hbase storage size

[liyang]

minor, fix display of hbase storage size


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ec099b3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ec099b3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ec099b3

Branch: refs/heads/master
Commit: 6ec099b31b448189930187612aa5be782f910d51
Parents: 72a4f0f
Author: Cheng Wang 
Authored: Fri Aug 18 20:35:25 2017 +0800
Committer: Roger Shi 
Committed: Sat Aug 19 21:09:59 2017 +0800

--
 .../apache/kylin/rest/service/CubeService.java  | 48 +---
 1 file changed, 41 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6ec099b3/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 16e94c7..f8ca38d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -24,7 +24,6 @@ import java.util.Collections;
 import java.util.Date;
 import java.util.EnumSet;
 import java.util.List;
-import java.util.WeakHashMap;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.KylinConfig;
@@ -41,6 +40,7 @@ import org.apache.kylin.engine.mr.CubingJob;
 import org.apache.kylin.job.exception.JobException;
 import org.apache.kylin.job.execution.DefaultChainedExecutable;
 import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
 import org.apache.kylin.metadata.draft.Draft;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
@@ -52,6 +52,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.rest.constant.Constant;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.msg.Message;
 import org.apache.kylin.rest.msg.MsgPicker;
 import org.apache.kylin.rest.request.MetricsRequest;
@@ -61,6 +62,7 @@ import org.apache.kylin.rest.security.AclPermission;
 import org.apache.kylin.rest.util.AclEvaluate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.security.access.AccessDeniedException;
@@ -68,6 +70,8 @@ import 
org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Component;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Lists;
 
 /**
@@ -76,14 +80,14 @@ import com.google.common.collect.Lists;
  * @author yangli9
  */
 @Component("cubeMgmtService")
-public class CubeService extends BasicService {
+public class CubeService extends BasicService implements InitializingBean {
 
 private static final Logger logger = 
LoggerFactory.getLogger(CubeService.class);
 
 public static final char[] VALID_CUBENAME = 
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_"
 .toCharArray();
 
-private WeakHashMap htableInfoCache = new 
WeakHashMap<>();
+protected Cache htableInfoCache = 
CacheBuilder.newBuilder().build();
 
 @Autowired
 @Qualifier("accessService")
@@ -101,7 +105,7 @@ public class CubeService extends BasicService {
 private AclEvaluate aclEvaluate;
 
 public List listAllCubes(final String cubeName, final String 
projectName, final String modelName,
-   boolean exactMatch) {
+boolean exactMatch) {
 List cubeInstances = null;
 ProjectInstance project = (null != projectName) ? 
getProjectManager().getProject(projectName) : null;
 
@@ -414,11 +418,12 @@ public class CubeService extends BasicService {
  * @throws IOException Exception when HTable resource is not closed 
correctly.
  */
 public HBaseResponse getHTableInfo(String tableName) throws IOException {
-if (htableInfoCache.containsKey(tableName)) {
-return htableInfoCache.get(tableName);
+HBaseResponse hr = htableInfoCache.getIfPresent(tableName);
+if (null != hr) {
+return hr;
 }
 
-

[21/50] [abbrv] kylin git commit: minor, add logs

[liyang]

minor, add logs


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a637d2a0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a637d2a0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a637d2a0

Branch: refs/heads/master
Commit: a637d2a0cea88d85a372bc449269a50a1502471f
Parents: 68ae79a
Author: Cheng Wang 
Authored: Tue Aug 22 16:18:17 2017 +0800
Committer: 成 
Committed: Tue Aug 22 16:46:53 2017 +0800

--
 .../main/java/org/apache/kylin/rest/service/CubeService.java| 1 +
 .../kylin/storage/hbase/util/HBaseRegionSizeCalculator.java | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a637d2a0/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index f8ca38d..fe8fb6c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -718,6 +718,7 @@ public class CubeService extends BasicService implements 
InitializingBean {
 @Override
 public void afterPropertiesSet() throws Exception {
 Broadcaster.getInstance(getConfig()).registerListener(new 
HTableInfoSyncListener(), "cube");
+logger.info("HTableInfoSyncListener is on.");
 }
 
 private class HTableInfoSyncListener extends Broadcaster.Listener {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a637d2a0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
--
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 1daca0a..5189628 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -104,8 +104,9 @@ public class HBaseRegionSizeCalculator {
 sizeMap.put(regionId, regionSizeBytes);
 countMap.put(regionId, new 
Pair<>(regionLoad.getStores(), regionLoad.getStorefiles()));
 
-// logger.info("Region " + regionLoad.getNameAsString()
-// + " has size " + regionSizeBytes);
+if (regionSizeBytes == 0L) {
+logger.info("Region " + 
regionLoad.getNameAsString() + " has size " + regionSizeBytes);
+}
 }
 }
 }

[07/50] [abbrv] kylin git commit: KYLIN-2760 Enhance ACL Logic

[liyang]

KYLIN-2760 Enhance ACL Logic


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a9f74c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a9f74c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a9f74c8

Branch: refs/heads/master
Commit: 7a9f74c8a0f70480f40c6e88e0aeb71824fd7884
Parents: fa7eb69
Author: tttMelody <245915...@qq.com>
Authored: Wed Aug 16 14:35:08 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Aug 16 16:04:09 2017 +0800

--
 .../org/apache/kylin/cube/CubeInstance.java |   6 +
 .../java/org/apache/kylin/job/JobInstance.java  |   2 +-
 .../kylin/metadata/model/DataModelDesc.java |   7 +-
 .../apache/kylin/rest/constant/Constant.java|  11 +-
 .../rest/controller/ProjectController.java  |  25 +---
 .../rest/controller/StreamingController.java|  37 +++---
 .../rest/controller2/CubeDescControllerV2.java  |  10 +-
 .../rest/controller2/ModelControllerV2.java |  23 ++--
 .../rest/controller2/ModelDescControllerV2.java |  18 +--
 .../rest/controller2/StreamingControllerV2.java |  33 ++---
 .../rest/controller2/TableControllerV2.java |   2 +-
 .../kylin/rest/service/AccessService.java   |   5 +
 .../apache/kylin/rest/service/CubeService.java  |  83 
 .../kylin/rest/service/DiagnosisService.java|  18 ++-
 .../kylin/rest/service/HybridService.java   |  16 ++-
 .../apache/kylin/rest/service/JobService.java   |  35 ++---
 .../kylin/rest/service/KafkaConfigService.java  |  25 ++--
 .../apache/kylin/rest/service/ModelService.java |  71 ++
 .../kylin/rest/service/ProjectService.java  |  29 +---
 .../apache/kylin/rest/service/QueryService.java |  10 +-
 .../kylin/rest/service/StreamingService.java|  22 ++--
 .../apache/kylin/rest/service/TableService.java |  27 ++--
 .../org/apache/kylin/rest/util/AclEvaluate.java | 131 +++
 .../org/apache/kylin/rest/util/AclUtil.java |  39 --
 .../org/apache/kylin/rest/bean/BeanTest.java|   7 +-
 .../rest/controller/AccessControllerTest.java   |  25 +++-
 26 files changed, 440 insertions(+), 277 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/7a9f74c8/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
--
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 4d205eb..4c57db8 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -36,6 +36,8 @@ import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.realization.CapabilityResult;
 import 
org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.apache.kylin.metadata.realization.IRealization;
@@ -385,6 +387,10 @@ public class CubeInstance extends RootPersistentEntity 
implements IRealization,
 return getDescriptor().getProject();
 }
 
+public ProjectInstance getProjectInstance() {
+return 
ProjectManager.getInstance(getConfig()).getProject(getProject());
+}
+
 @Override
 public int getSourceType() {
 return getModel().getRootFactTable().getTableDesc().getSourceType();

http://git-wip-us.apache.org/repos/asf/kylin/blob/7a9f74c8/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
--
diff --git a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java 
b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
index 3778834..b94 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
@@ -167,7 +167,7 @@ public class JobInstance extends RootPersistentEntity 
implements Comparablehttp://git-wip-us.apache.org/repos/asf/kylin/blob/7a9f74c8/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index f82c996..2037420 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -40,6 +40,7 

[28/50] [abbrv] kylin git commit: KYLIN-2809 update calcite to support "+" as string concat

[liyang]

KYLIN-2809 update calcite to support "+" as string concat


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d059c05e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d059c05e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d059c05e

Branch: refs/heads/master
Commit: d059c05e7a064c6c6bc861fb3e399985ed968177
Parents: d873ca1
Author: Roger Shi 
Authored: Thu Aug 24 15:28:10 2017 +0800
Committer: Roger Shi 
Committed: Thu Aug 24 15:46:12 2017 +0800

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d059c05e/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 5b00798..d82a7f4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,7 +67,7 @@
 
 
 
-1.13.0-kylin-r1-SNAPSHOT
+1.13.0-kylin-r2-SNAPSHOT
 1.10.0
 
 

[19/50] [abbrv] kylin git commit: KYLIN-2801 Make default precision and scale in DataType (for hive) configurable

[liyang]

KYLIN-2801 Make default precision and scale in DataType (for hive) configurable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a3d42f11
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a3d42f11
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a3d42f11

Branch: refs/heads/master
Commit: a3d42f11924c734b40a7b61fbf6f2f8bac3caa88
Parents: 46cd881
Author: Hongbin Ma 
Authored: Tue Aug 22 14:12:27 2017 +0800
Committer: Hongbin Ma 
Committed: Tue Aug 22 16:00:14 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java| 41 
 .../kylin/metadata/datatype/DataType.java   | 24 ++--
 2 files changed, 54 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a3d42f11/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 0f5f3a9..b622825 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -598,6 +598,47 @@ abstract public class KylinConfigBase implements 
Serializable {
 return 
getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
 }
 
+public int getDefaultVarcharPrecision() {
+int v = 
Integer.parseInt(getOptional("kylin.source.hive.default-varchar-precision", 
"256"));
+if (v < 1) {
+return 256;
+} else if (v > 65355) {
+return 65535;
+} else {
+return v;
+}
+}
+
+public int getDefaultCharPrecision() {
+//at most 255 according to 
https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-CharcharChar
+int v = 
Integer.parseInt(getOptional("kylin.source.hive.default-char-precision", 
"255"));
+if (v < 1) {
+return 255;
+} else if (v > 255) {
+return 255;
+} else {
+return v;
+}
+}
+
+public int getDefaultDecimalPrecision() {
+int v = 
Integer.parseInt(getOptional("kylin.source.hive.default-decimal-precision", 
"19"));
+if (v < 1) {
+return 19;
+} else {
+return v;
+}
+}
+
+public int getDefaultDecimalScale() {
+int v = 
Integer.parseInt(getOptional("kylin.source.hive.default-decimal-scale", "4"));
+if (v < 1) {
+return 4;
+} else {
+return v;
+}
+}
+
 // 

 // SOURCE.KAFKA
 // 


http://git-wip-us.apache.org/repos/asf/kylin/blob/a3d42f11/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index 66ff581..13cc2be 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -31,6 +31,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.BytesSerializer;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.measure.MeasureTypeFactory;
@@ -177,21 +178,22 @@ public class DataType implements Serializable {
 }
 }
 
-// FIXME 256 for unknown string precision
-if ((name.equals("char") || name.equals("varchar")) && precision == 
-1) {
-precision = 256; // to save memory at frontend, e.g. tableau will
- // allocate memory according to this
+if (precision == -1) {
+// FIXME 256 for unknown string precision
+
+// why 256(255) as default? 
+// to save memory at frontend, e.g. tableau will
+// allocate memory according to this
 if (name.equals("char")) {
-precision -= 1; //at most 255 according to 
https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-CharcharChar
+precision = 
KylinConfig.getInstanceFromEnv().getDefaultCharPrecision();
+} else if (name.equals("varchar")) 

[16/50] [abbrv] kylin git commit: KYLIN-2575 code refactors

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java 
b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
index f2894e7..ce50122 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
@@ -56,6 +56,10 @@ public class ColumnRowType {
 return columns.get(index);
 }
 
+public TblColRef getColumnByName(String columnName) {
+return getColumnByIndexNullable(getIndexByName(columnName));
+}
+
 public int getIndexByName(String columnName) {
 for (int i = 0; i < columns.size(); i++) {
 if (columns.get(i).getName().equals(columnName)) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index e7b09a3..8e93659 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -158,7 +158,7 @@ public class OLAPProjectRel extends Project implements 
OLAPRel {
 RexCall call = (RexCall) rexNode;
 column = translateRexCall(call, inputColumnRowType, fieldName, 
sourceCollector);
 } else {
-throw new IllegalStateException("Unsupport RexNode " + rexNode);
+throw new IllegalStateException("Unsupported RexNode " + rexNode);
 }
 return column;
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
--
diff --git 
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java 
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index b583291..7801891 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -238,7 +238,7 @@ public class OLAPTableScan extends TableScan implements 
OLAPRel, EnumerableRel {
 TableRef tableRef = TblColRef.tableForUnknownModel(this.alias, 
olapTable.getSourceTable());
 
 List columns = new ArrayList();
-for (ColumnDesc sourceColumn : olapTable.getExposedColumns()) {
+for (ColumnDesc sourceColumn : olapTable.getSourceColumns()) {
 TblColRef colRef = TblColRef.columnForUnknownModel(tableRef, 
sourceColumn);
 columns.add(colRef);
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/aed840f9/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java 
b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
index fe897e9..5387a74 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
@@ -41,6 +41,7 @@ public class OLAPSchema extends AbstractSchema {
 private KylinConfig config;
 private String projectName;
 private String schemaName;
+private boolean exposeMore;
 private StorageURL storageUrl;
 private String starSchemaUrl;
 private String starSchemaUser;
@@ -54,9 +55,10 @@ public class OLAPSchema extends AbstractSchema {
 this.starSchemaPassword = config.getHivePassword();
 }
 
-public OLAPSchema(String project, String schemaName) {
+public OLAPSchema(String project, String schemaName, boolean exposeMore) {
 this.projectName = ProjectInstance.getNormalizedProjectName(project);
 this.schemaName = schemaName;
+this.exposeMore = exposeMore;
 init();
 }
 
@@ -66,19 +68,21 @@ public class OLAPSchema extends 

[12/50] [abbrv] kylin git commit: minor, fix query export garbled when results having Chinese

[liyang]

minor, fix query export garbled when results having Chinese



Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8906d133
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8906d133
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8906d133

Branch: refs/heads/master
Commit: 8906d133f430254ed4a8c1f6f7b989b0c4ca2c09
Parents: 1c8de93
Author: Jiatao Tao <245915...@qq.com>
Authored: Mon Aug 21 02:08:08 2017 -0500
Committer: GitHub 
Committed: Mon Aug 21 02:08:08 2017 -0500

--
 .../apache/kylin/rest/controller2/QueryControllerV2.java  | 10 --
 1 file changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8906d133/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
 
b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
index 5809d6e..1aa72ba 100644
--- 
a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
+++ 
b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
@@ -19,6 +19,9 @@
 package org.apache.kylin.rest.controller2;
 
 import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
 import java.sql.SQLException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -159,7 +162,7 @@ public class QueryControllerV2 extends BasicController {
 "application/vnd.apache.kylin-v2+json" }, consumes = 
MediaType.APPLICATION_FORM_URLENCODED_VALUE)
 @ResponseBody
 public void downloadQueryResultV2(@PathVariable String format, SQLRequest 
sqlRequest,
-HttpServletResponse response) {
+HttpServletResponse response) throws IOException {
 
 SQLResponse result = queryService.doQueryWithCache(sqlRequest);
 
@@ -170,8 +173,11 @@ public class QueryControllerV2 extends BasicController {
 ICsvListWriter csvWriter = null;
 
 try {
-csvWriter = new CsvListWriter(response.getWriter(), 
CsvPreference.STANDARD_PREFERENCE);
+//Add a BOM for Excel
+Writer writer = new OutputStreamWriter(response.getOutputStream(), 
StandardCharsets.UTF_8);
+writer.write('\uFEFF');
 
+csvWriter = new CsvListWriter(writer, 
CsvPreference.STANDARD_PREFERENCE);
 List headerList = new ArrayList();
 
 for (SelectedColumnMeta column : result.getColumnMetas()) {

[23/50] [abbrv] kylin git commit: minor, only apply adjustSqlDigest to involved aggregations

[liyang]

minor, only apply adjustSqlDigest to involved aggregations


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d85f6ab0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d85f6ab0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d85f6ab0

Branch: refs/heads/master
Commit: d85f6ab0130e3c1962c9dc770e06f178d3d85c32
Parents: f26b1ab
Author: Roger Shi 
Authored: Tue Aug 22 17:42:37 2017 +0800
Committer: Roger Shi 
Committed: Tue Aug 22 17:55:53 2017 +0800

--
 .../org/apache/kylin/cube/CubeCapabilityChecker.java |  5 +
 .../extendedcolumn/ExtendedColumnMeasureType.java| 10 +-
 .../org/apache/kylin/measure/raw/RawMeasureType.java | 10 +-
 .../apache/kylin/measure/topn/TopNMeasureType.java   | 15 ++-
 .../kylin/metadata/realization/CapabilityResult.java |  8 
 .../apache/kylin/metadata/realization/SQLDigest.java | 10 --
 .../apache/kylin/storage/hbase/ITStorageTest.java|  4 +++-
 .../kylin/query/enumerator/OLAPEnumerator.java   |  1 -
 .../org/apache/kylin/query/relnode/OLAPContext.java  |  5 -
 .../org/apache/kylin/query/routing/QueryRouter.java  |  6 ++
 10 files changed, 66 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java 
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 61516d2..47f268a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -127,6 +127,11 @@ public class CubeCapabilityChecker {
 public double suggestCostMultiplier() {
 return 100;
 }
+
+@Override
+public MeasureDesc getInvolvedMeasure() {
+return null;
+}
 });
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
 
b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
index de5ee25..c95cd1c 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
@@ -95,6 +95,9 @@ public class ExtendedColumnMeasureType extends 
MeasureType {
 @Override
 public void adjustSqlDigest(List measureDescs, SQLDigest 
sqlDigest) {
 for (MeasureDesc measureDesc : measureDescs) {
+if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
+continue;
+}
 FunctionDesc extendColumnFunc = measureDesc.getFunction();
 List hosts = getExtendedColumnHosts(extendColumnFunc);
 TblColRef extended = getExtendedColumn(extendColumnFunc);
@@ -111,7 +114,7 @@ public class ExtendedColumnMeasureType extends 
MeasureType {
 }
 
 @Override
-public CapabilityResult.CapabilityInfluence 
influenceCapabilityCheck(Collection unmatchedDimensions, 
Collection unmatchedAggregations, SQLDigest digest, MeasureDesc 
measureDesc) {
+public CapabilityResult.CapabilityInfluence 
influenceCapabilityCheck(Collection unmatchedDimensions, 
Collection unmatchedAggregations, SQLDigest digest, final 
MeasureDesc measureDesc) {
 TblColRef extendedCol = getExtendedColumn(measureDesc.getFunction());
 
 if (!unmatchedDimensions.contains(extendedCol)) {
@@ -129,6 +132,11 @@ public class ExtendedColumnMeasureType extends 
MeasureType {
 public double suggestCostMultiplier() {
 return 0.9;
 }
+
+@Override
+public MeasureDesc getInvolvedMeasure() {
+return measureDesc;
+}
 };
 }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/d85f6ab0/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java 
b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 31f35e9..eee8fa6 100644
--- 

[04/50] [abbrv] kylin git commit: #1949, fix NPE when discarding model check job (#1952)

[liyang]

#1949, fix NPE when discarding model check job (#1952)



Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c487fe49
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c487fe49
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c487fe49

Branch: refs/heads/master
Commit: c487fe4990473afa11d30a2e16ceb133e3f25b6d
Parents: 2eca43c
Author: 成 
Authored: Wed Aug 16 12:05:20 2017 +0800
Committer: luguosheng1314 <550175...@qq.com>
Committed: Wed Aug 16 12:05:20 2017 +0800

--
 .../apache/kylin/rest/service/JobService.java   | 127 +--
 1 file changed, 63 insertions(+), 64 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c487fe49/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 6451d66..5dafa08 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -18,18 +18,12 @@
 
 package org.apache.kylin.rest.service;
 
-import java.io.IOException;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-
-import javax.annotation.Nullable;
-
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.directory.api.util.Strings;
 import org.apache.kylin.common.KylinConfig;
@@ -72,12 +66,16 @@ import 
org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Component;
 
-import com.google.common.base.Function;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
 
 /**
  * @author ysong1
@@ -158,22 +156,22 @@ public class JobService extends BasicService implements 
InitializingBean {
 Message msg = MsgPicker.getMsg();
 
 switch (status) {
-case DISCARDED:
-return ExecutableState.DISCARDED;
-case ERROR:
-return ExecutableState.ERROR;
-case FINISHED:
-return ExecutableState.SUCCEED;
-case NEW:
-return ExecutableState.READY;
-case PENDING:
-return ExecutableState.READY;
-case RUNNING:
-return ExecutableState.RUNNING;
-case STOPPED:
-return ExecutableState.STOPPED;
-default:
-throw new 
BadRequestException(String.format(msg.getILLEGAL_EXECUTABLE_STATE(), status));
+case DISCARDED:
+return ExecutableState.DISCARDED;
+case ERROR:
+return ExecutableState.ERROR;
+case FINISHED:
+return ExecutableState.SUCCEED;
+case NEW:
+return ExecutableState.READY;
+case PENDING:
+return ExecutableState.READY;
+case RUNNING:
+return ExecutableState.RUNNING;
+case STOPPED:
+return ExecutableState.STOPPED;
+default:
+throw new 
BadRequestException(String.format(msg.getILLEGAL_EXECUTABLE_STATE(), status));
 }
 }
 
@@ -181,30 +179,30 @@ public class JobService extends BasicService implements 
InitializingBean {
 Message msg = MsgPicker.getMsg();
 
 switch (timeFilter) {
-case LAST_ONE_DAY:
-calendar.add(Calendar.DAY_OF_MONTH, -1);
-return calendar.getTimeInMillis();
-case LAST_ONE_WEEK:
-calendar.add(Calendar.WEEK_OF_MONTH, -1);
-return calendar.getTimeInMillis();
-case LAST_ONE_MONTH:
-calendar.add(Calendar.MONTH, -1);
-return calendar.getTimeInMillis();
-case LAST_ONE_YEAR:
-calendar.add(Calendar.YEAR, -1);
-return 

[02/50] [abbrv] kylin git commit: KYLIN-2783 merge AggregationGroupScheduler into CuboidScheduler

[liyang]

KYLIN-2783 merge AggregationGroupScheduler into CuboidScheduler


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8bab3df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8bab3df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8bab3df

Branch: refs/heads/master
Commit: c8bab3df33edd5dc146a02f94caccd0d810b68bd
Parents: 5008866
Author: Li Yang 
Authored: Sun Aug 13 16:34:22 2017 +0800
Committer: Roger Shi 
Committed: Mon Aug 14 09:35:09 2017 +0800

--
 .../org/apache/kylin/common/KylinVersion.java   |  24 ++-
 .../apache/kylin/common/KylinVersionTest.java   |  19 +-
 .../cube/cuboid/AggregationGroupScheduler.java  | 126 
 .../kylin/cube/cuboid/CuboidScheduler.java  |   4 +
 .../cube/cuboid/DefaultCuboidScheduler.java | 191 ++-
 .../kylin/cube/model/AggregationGroup.java  |  10 +-
 .../org/apache/kylin/cube/model/SelectRule.java |   6 +-
 7 files changed, 187 insertions(+), 193 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 960bafe..c6913fc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -37,10 +37,11 @@ public class KylinVersion implements Comparable {
 private static final String COMMIT_SHA1_v15 = "commit_SHA1";
 private static final String COMMIT_SHA1_v13 = "commit.sha1";
 
-public int major;
-public int minor;
-public int revision;
-public boolean isSnapshot;
+public final int major;
+public final int minor;
+public final int revision;
+public final int internal;
+public final boolean isSnapshot;
 
 public KylinVersion(String version) {
 
@@ -58,12 +59,13 @@ public class KylinVersion implements Comparable {
 
 major = Integer.parseInt(splits[0]);
 minor = Integer.parseInt(splits[1]);
-revision = splits.length < 3 ? 0 : Integer.parseInt(splits[2]);
+revision = splits.length > 2 ? Integer.parseInt(splits[2]) : 0;
+internal = splits.length > 3 ? Integer.parseInt(splits[3]) : 0;
 }
 
 @Override
 public String toString() {
-return "" + major + "." + minor + "." + revision;
+return "" + major + "." + minor + "." + revision + "." + internal + 
(isSnapshot ? "-SNAPSHOT" : "");
 }
 
 @Override
@@ -83,13 +85,17 @@ public class KylinVersion implements Comparable {
 if (comp != 0)
 return comp;
 
+comp = this.internal - v.internal;
+if (comp != 0)
+return comp;
+
 return (this.isSnapshot ? 0 : 1) - (v.isSnapshot ? 0 : 1);
 }
 
 /**
  * Require MANUAL updating kylin version per ANY upgrading.
  */
-private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("2.1.0");
+private static final KylinVersion CURRENT_KYLIN_VERSION = new 
KylinVersion("2.1.0.20403");
 
 private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");
 
@@ -125,6 +131,10 @@ public class KylinVersion implements Comparable {
 public static boolean isBefore200(String ver) {
 return new KylinVersion(ver).compareTo(VERSION_200) < 0;
 }
+
+public static int compare(String v1, String v2) {
+return new KylinVersion(v1).compareTo(new KylinVersion(v2));
+}
 
 public static void main(String[] args) {
 System.out.println(getKylinClientInformation());

http://git-wip-us.apache.org/repos/asf/kylin/blob/c8bab3df/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
--
diff --git 
a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java 
b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
index 96b3189..6e903c8 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinVersionTest.java
@@ -37,4 +37,21 @@ public class KylinVersionTest {
 Assert.assertEquals(1, ver1.minor);
 Assert.assertEquals(0, ver1.revision);
 }
-}
+
+@Test
+public void testToString() {
+KylinVersion ver1 = new KylinVersion("2.1.7.321");
+Assert.assertEquals(2, ver1.major);
+Assert.assertEquals(1, ver1.minor);
+Assert.assertEquals(7, ver1.revision);
+Assert.assertEquals(321, ver1.internal);
+  

[06/50] [abbrv] kylin git commit: KYLIN-2760 Enhance ACL Logic

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/7a9f74c8/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
--
diff --git 
a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java 
b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index 751d014..30ee99c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -43,11 +43,11 @@ import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableExtDesc;
 import org.apache.kylin.metadata.streaming.StreamingConfig;
-import org.apache.kylin.rest.constant.Constant;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.msg.Message;
 import org.apache.kylin.rest.msg.MsgPicker;
 import org.apache.kylin.rest.response.TableDescResponse;
+import org.apache.kylin.rest.util.AclEvaluate;
 import org.apache.kylin.source.ISourceMetadataExplorer;
 import org.apache.kylin.source.SourceFactory;
 import org.apache.kylin.source.hive.cardinality.HiveColumnCardinalityJob;
@@ -57,7 +57,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Component;
 
@@ -88,27 +87,34 @@ public class TableService extends BasicService {
 @Qualifier("kafkaMgmtService")
 private KafkaConfigService kafkaConfigService;
 
+@Autowired
+private AclEvaluate aclEvaluate;
+
 public List getTableDescByProject(String project, boolean 
withExt) throws IOException {
+aclEvaluate.checkProjectReadPermission(project);
 List tables = 
getProjectManager().listDefinedTables(project);
 if (null == tables) {
 return Collections.emptyList();
 }
 if (withExt) {
+aclEvaluate.checkProjectWritePermission(project);
 tables = cloneTableDesc(tables, project);
 }
 return tables;
 }
 
 public TableDesc getTableDescByName(String tableName, boolean withExt, 
String prj) {
+aclEvaluate.checkProjectReadPermission(prj);
 TableDesc table = getMetadataManager().getTableDesc(tableName, prj);
 if (withExt) {
+aclEvaluate.checkProjectWritePermission(prj);
 table = cloneTableDesc(table, prj);
 }
 return table;
 }
 
-@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
 public String[] loadHiveTablesToProject(String[] tables, String project) 
throws Exception {
+aclEvaluate.checkProjectWritePermission(project);
 // de-dup
 SetMultimap db2tables = LinkedHashMultimap.create();
 for (String fullTableName : tables) {
@@ -176,6 +182,7 @@ public class TableService extends BasicService {
 }
 
 public Map loadHiveTables(String[] tableNames, String 
project, boolean isNeedProfile) throws Exception {
+aclEvaluate.checkProjectWritePermission(project);
 String submitter = 
SecurityContextHolder.getContext().getAuthentication().getName();
 Map result = new HashMap();
 
@@ -198,6 +205,7 @@ public class TableService extends BasicService {
 }
 
 public Map unloadHiveTables(String[] tableNames, String 
project) throws IOException {
+aclEvaluate.checkProjectWritePermission(project);
 Set unLoadSuccess = Sets.newHashSet();
 Set unLoadFail = Sets.newHashSet();
 Map result = new HashMap();
@@ -231,8 +239,8 @@ public class TableService extends BasicService {
  * @param project
  * @return
  */
-@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
 public boolean unLoadHiveTable(String tableName, String project) throws 
IOException {
+aclEvaluate.checkProjectWritePermission(project);
 Message msg = MsgPicker.getMsg();
 
 boolean rtn = false;
@@ -266,9 +274,9 @@ public class TableService extends BasicService {
 KafkaConfig kafkaConfig = null;
 try {
 config = 
streamingService.getStreamingManager().getStreamingConfig(tableName);
-kafkaConfig = kafkaConfigService.getKafkaConfig(tableName);
-streamingService.dropStreamingConfig(config);
-kafkaConfigService.dropKafkaConfig(kafkaConfig);
+kafkaConfig = kafkaConfigService.getKafkaConfig(tableName, 
project);
+streamingService.dropStreamingConfig(config, project);
+

[13/50] [abbrv] kylin git commit: minor, unknown precision for char should be 255

[liyang]

minor, unknown precision for char should be 255


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/909628ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/909628ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/909628ff

Branch: refs/heads/master
Commit: 909628ff6d128639eb813d616fe7281295378619
Parents: 788c731
Author: Hongbin Ma 
Authored: Mon Aug 21 22:21:25 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Aug 21 22:55:59 2017 +0800

--
 .../org/apache/kylin/metadata/datatype/DataType.java | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/909628ff/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
--
diff --git 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index 83b2391..66ff581 100644
--- 
a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ 
b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -164,14 +164,16 @@ public class DataType implements Serializable {
 try {
 n = Integer.parseInt(parts[i]);
 } catch (NumberFormatException e) {
-throw new IllegalArgumentException("bad data type -- " + 
datatype + ", precision/scale not numeric");
+throw new IllegalArgumentException(
+"bad data type -- " + datatype + ", 
precision/scale not numeric");
 }
 if (i == 0)
 precision = n;
 else if (i == 1)
 scale = n;
 else
-throw new IllegalArgumentException("bad data type -- " + 
datatype + ", too many precision/scale parts");
+throw new IllegalArgumentException(
+"bad data type -- " + datatype + ", too many 
precision/scale parts");
 }
 }
 
@@ -179,6 +181,9 @@ public class DataType implements Serializable {
 if ((name.equals("char") || name.equals("varchar")) && precision == 
-1) {
 precision = 256; // to save memory at frontend, e.g. tableau will
  // allocate memory according to this
+if (name.equals("char")) {
+precision -= 1; //at most 255 according to 
https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-CharcharChar
+}
 }
 
 // FIXME (19,4) for unknown decimal precision
@@ -217,7 +222,7 @@ public class DataType implements Serializable {
 public boolean isTimeFamily() {
 return DATETIME_FAMILY.contains(name) && !isDate();
 }
-
+
 public boolean isDate() {
 return name.equals("date");
 }

[25/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as default metric framework

[liyang]

KYLIN-2776 Using dropwizard as default metric framework


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8e078847
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8e078847
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8e078847

Branch: refs/heads/master
Commit: 8e078847a42b4f7853a2c1a75afa405c09c4ea7b
Parents: d85f6ab
Author: yiming.xu <100650...@qq.com>
Authored: Wed Aug 23 13:12:58 2017 +0800
Committer: Hongbin Ma 
Committed: Wed Aug 23 13:49:23 2017 +0800

--
 core-common/pom.xml |  17 +
 .../apache/kylin/common/KylinConfigBase.java|  20 +
 .../kylin/common/metrics/common/Metrics.java| 162 ++
 .../common/metrics/common/MetricsConstant.java  |  38 ++
 .../common/metrics/common/MetricsFactory.java   |  62 +++
 .../common/metrics/common/MetricsScope.java |  33 ++
 .../common/metrics/common/MetricsVariable.java  |  27 +
 .../kylin/common/metrics/common/Metricss.java   |  40 ++
 .../metrics/metrics2/CodahaleMetrics.java   | 508 +++
 .../metrics/metrics2/CodahaleReporter.java  |  31 ++
 .../metrics2/ConsoleMetricsReporter.java|  51 ++
 .../metrics/metrics2/JmxMetricsReporter.java|  55 ++
 .../metrics2/JsonFileMetricsReporter.java   | 136 +
 .../metrics2/KylinObjectNameFactory.java|  68 +++
 .../metrics2/MetricVariableRatioGauge.java  |  47 ++
 .../metrics/metrics2/Metrics2Reporter.java  |  60 +++
 .../metrics/metrics2/MetricsReporting.java  |  26 +
 .../common/metrics/perflog/IPerfLogger.java |  48 ++
 .../common/metrics/perflog/PerfLogger.java  | 158 ++
 .../metrics/perflog/PerfLoggerFactory.java  |  56 ++
 .../metrics/perflog/SimplePerfLogger.java   |  73 +++
 pom.xml |   3 +-
 .../kylin/rest/init/InitialTaskManager.java |   4 +-
 .../kylin/rest/metrics/QueryMetrics2Facade.java | 106 
 .../apache/kylin/rest/service/QueryService.java |   2 +
 .../kylin/rest/metrics/QueryMetrics2Test.java   | 128 +
 26 files changed, 1957 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 8852743..d58de36 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -68,6 +68,23 @@
 junit
 test
 
+
+
+io.dropwizard.metrics
+metrics-jvm
+${dropwizard.version}
+
 
+
+io.dropwizard.metrics
+metrics-json
+${dropwizard.version}
+
+
+
+com.github.joshelser
+
dropwizard-metrics-hadoop-metrics2-reporter
+0.1.2
+
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b622825..99b9bb2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1173,4 +1173,24 @@ abstract public class KylinConfigBase implements 
Serializable {
 public boolean isWebCrossDomainEnabled() {
 return 
Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
 }
+
+/**
+ * metric
+ */
+public String getCoadhaleMetricReportClassesName() {
+return getOptional("kylin.metric.codahale-metric-report-classes",
+
"org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter");
+}
+
+public String getMetricFileLocation() {
+return getOptional("kylin.metric.file.location", "/tmp/report.json");
+}
+
+public Long getJsonFileMetricsReporterInterval() {
+return 
Long.parseLong(getOptional("kylin.metric.json-file-metric-reporter.interval", 
"5000"));
+}
+
+public String getPerfLoggerClassName() {
+return getOptional("kylin.metric.perf-logger.class", 
"org.apache.kylin.common.metrics.perflog.PerfLogger");
+}
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java 

[29/50] [abbrv] kylin git commit: minor, fix some files license header

[liyang]

minor, fix some files license header


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1fff4214
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1fff4214
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1fff4214

Branch: refs/heads/master
Commit: 1fff42146aa539b599802f9b19de28dc87d05fad
Parents: d059c05
Author: Roger Shi 
Authored: Fri Aug 25 17:31:19 2017 +0800
Committer: Roger Shi 
Committed: Fri Aug 25 17:36:24 2017 +0800

--
 .../calcite/prepare/CalcitePrepareImpl.java | 34 
 .../calcite/sql2rel/SqlToRelConverter.java  | 34 
 .../java/org/apache/calcite/tools/Programs.java | 34 
 3 files changed, 42 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1fff4214/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java 
b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
index b1ce7ea..7b4c549 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
@@ -1,26 +1,20 @@
 /*
- * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * http://kyligence.io
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
- * This software is the confidential and proprietary information of
- * Kyligence Inc. ("Confidential Information"). You shall not disclose
- * such Confidential Information and shall use it only in accordance
- * with the terms of the license agreement you entered into with
- * Kyligence Inc.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
 package org.apache.calcite.prepare;
 
 import static org.apache.calcite.util.Static.RESOURCE;

http://git-wip-us.apache.org/repos/asf/kylin/blob/1fff4214/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 18aadef..f10ba96 100644
--- 
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ 
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -1,26 +1,20 @@
 /*
- * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * http://kyligence.io
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
- * This software is the confidential and proprietary information of
- * Kyligence Inc. ("Confidential Information"). You shall not disclose
- 

[24/50] [abbrv] kylin git commit: KYLIN-2776 Using dropwizard as default metric framework

[liyang]

http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java
--
diff --git 
a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java 
b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java
new file mode 100644
index 000..5e3e00b
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetrics2Test.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.metrics;
+
+import static 
org.apache.kylin.common.metrics.common.Metricss.buildCubeMetricPrefix;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+import org.apache.kylin.common.metrics.common.Metrics;
+import org.apache.kylin.common.metrics.common.MetricsConstant;
+import org.apache.kylin.common.metrics.common.MetricsFactory;
+import org.apache.kylin.common.metrics.common.Metricss;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.rest.service.ServiceTestBase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.codahale.metrics.Snapshot;
+
+public class QueryMetrics2Test extends ServiceTestBase {
+
+private static MBeanServer mBeanServer;
+private static ObjectName objectName;
+
+@Before
+public void setup() throws Exception {
+super.setup();
+
+objectName = new ObjectName("Hadoop:service=Kylin,name=Server_Total");
+}
+
+@Test
+public void testQueryMetrics() throws Exception {
+System.setProperty("kylin.server.query-metrics-enabled", "true");
+QueryMetrics2Facade.init();
+
+SQLRequest sqlRequest = new SQLRequest();
+sqlRequest.setSql("select * from TEST_KYLIN_FACT");
+sqlRequest.setProject("default");
+
+SQLResponse sqlResponse = new SQLResponse();
+sqlResponse.setDuration(10);
+sqlResponse.setCube("test_cube");
+sqlResponse.setIsException(false);
+sqlResponse.setTotalScanCount(100);
+List list1 = new ArrayList<>();
+list1.add("111");
+list1.add("112");
+List list2 = new ArrayList<>();
+list2.add("111");
+list2.add("112");
+List results = new ArrayList<>();
+results.add(list1);
+results.add(list2);
+sqlResponse.setResults(results);
+sqlResponse.setStorageCacheUsed(true);
+Metrics metrics = MetricsFactory.getInstance();
+QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
+String prefix = buildCubeMetricPrefix(sqlRequest.getProject(), 
sqlResponse.getCube().replace("=", "->"));
+Thread.sleep(2000);
+
+Assert.assertEquals(1L,
+metrics.getCounter(Metricss.buildMetricName(prefix, 
MetricsConstant.QUERY_COUNT)).getCount());
+Assert.assertEquals(1L,
+metrics.getCounter(Metricss.buildMetricName(prefix, 
MetricsConstant.QUERY_SUCCESS_COUNT)).getCount());
+Assert.assertEquals(0L,
+metrics.getCounter(Metricss.buildMetricName(prefix, 
MetricsConstant.QUERY_FAIL_COUNT)).getCount());
+Assert.assertEquals(1L,
+metrics.getCounter(Metricss.buildMetricName(prefix, 
MetricsConstant.QUERY_CACHE_COUNT)).getCount());
+//
+Snapshot  queryScanSnapshot= metrics
+.getHistogram(Metricss.buildMetricName(prefix, 
MetricsConstant.QUERY_SCAN_ROWCOUNT)).getSnapshot();
+Assert.assertEquals(100.0, queryScanSnapshot.getMean(), 0);
+Assert.assertEquals(100.0, queryScanSnapshot.getMax(), 0);
+Assert.assertEquals(100.0, queryScanSnapshot.getMin(), 0);
+
+Snapshot queryResultSnapshot = metrics
+.getHistogram(Metricss.buildMetricName(prefix, 
MetricsConstant.QUERY_RESULT_ROWCOUNT)).getSnapshot();
+
+Assert.assertEquals(2.0, queryResultSnapshot.getMean(), 0);
+Assert.assertEquals(2.0, 

svn commit: r1806793 [2/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.

[lidong]

Modified: kylin/site/docs21/howto/howto_backup_metadata.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs21/howto/howto_backup_metadata.html?rev=1806793=1806792=1806793=diff
==
--- kylin/site/docs21/howto/howto_backup_metadata.html (original)
+++ kylin/site/docs21/howto/howto_backup_metadata.html Thu Aug 31 14:01:54 2017
@@ -335,11 +335,7 @@
 
   
 
-  
-
-  
-
-  FAQ  
+  Technical Concepts   
   
   
 
 
@@ -359,7 +355,11 @@
 
   
 
-  Events and 
Conferences  
+  
+
+  
+
+  Terminology  
   
 
 
@@ -375,7 +375,13 @@
 
   
 
-  Community Best 
Practices  
+  
+
+  
+
+  
+
+  FAQ  
   
 
 
@@ -395,11 +401,7 @@
 
   
 
-  
-
-  
-
-  Terminology  
+  Events and 
Conferences  
   
 
 
@@ -415,9 +417,7 @@
 
   
 
-  
-
-  Technical Concepts   
   
+  Community Best 
Practices  
   
 
 
@@ -678,6 +678,8 @@
   
 
   
+
+  
 
 
 
@@ -1054,7 +1056,53 @@
 
   
 
-  Kylin Cube Creation  

+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Web Interface  
   
 
 
@@ -1132,11 +1180,7 @@
 
   
 
-  
-
-  
-
-  Kylin Cube Build and Job 
Monitoring  
+  Cube Wizard  
   
 
 
@@ -1218,11 +1262,7 @@
 
   
 
-  
-
-  
-
-  Build Cube with Spark 
(beta)  
+  Cube Build and Job 
Monitoring  
   
 
 
@@ -1296,7 +1336,7 @@
 
   
 
-  Kylin Cube Permission  
+  Cube Permission  
   
 
 
@@ -1382,6 +1422,54 @@
 
   
 
+  Build Cube with Spark 
(beta)  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
   
 
   
@@ -1418,11 +1506,29 @@
 
   
 
-  Kylin Web Interface  
+  Cube Build Tuning
  
   
 
 
 
+
+
+
+
+
+  
+
+
+Integration
+
+   
+
+
+
+
+
+
+
   
   
 
@@ -1520,9 +1626,21 @@
 
   
 
+  Kylin ODBC Driver  
+  
+
+
+
+  
+  
+
   
 
   
+
+
+
+  
 
   
 
@@ -1532,7 +1650,15 @@
 
   
 
-  Tableau 8  
+  
+
+  
+
+  
+
+  
+
+  Kylin JDBC Driver  
   
 
 
@@ -1648,9 +1774,7 @@
 
   
 
-  
-
-  Tableau 9  
+  Tableau 8  
   
 
 
@@ -1754,7 +1878,23 @@
 
   
 
-  MS Excel and Power BI  
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Tableau 9  
   
 
 
@@ -1854,7 +1994,13 @@
 
   
 
-  Kylin ODBC Driver  
+  
+
+  
+
+  
+
+  MS Excel and Power BI  
   
 
 
@@ -1944,7 +2090,15 @@
 
   
 
-  Connect from Apache Flink  
+  
+
+  
+
+  
+
+  
+
+  MicroStrategy  
   
 
 
@@ -2054,7 +2208,9 @@
 
   
 
-  Connect from SQuirreL  
+  
+
+  SQuirreL  
   
 
 
@@ -2138,7 +2294,13 @@
 
   
 
-  Cube Build Tuning Step by 
Step  
+  
+
+  
+
+  
+
+  Apache Flink  
   
 
 
@@ -2230,7 +2392,7 @@
 
   
 
-  Connect from Apache Hue  
+  Apache Hue  
   
 
 
@@ -2276,24 +2438,6 @@
 
   
 
-  Build Cube with RESTful 
API  
-  
-
-
-
-  
-  
-
-  
-
-  
-
-
-
-  
-
-  
-
   
 
   
@@ -2314,19 +2458,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API in 
Javascript  
+  Use RESTful API  
   
 
 
@@ -2354,27 +2486,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API  
+  Build Cube with API  

   
 
 
@@ -2504,7 +2616,7 @@
 
   
 
-  Cleanup Storage (HDFS & 
HBase)  
+  Cleanup Storage  
   
 
 
@@ -2536,7 +2648,7 @@
 
   
 
-  Use JDBC Driver  
+  Kylin JDBC Driver  
   
 
 
@@ -2612,7 +2724,7 @@
 
   
 
-  Enable Security with LDAP and 
SSO  
+  Secure with LDAP and 
SSO  
   
 
 
@@ -2656,7 +2768,7 @@
 
   
 
-  Use Beeline for Hive 
Commands  
+  Use Beeline for Hive 
 
   
 
 
@@ -2696,7 +2808,7 @@
 
   
 
-  How to Update HBase 
Coprocessor  
+  Update Coprocessor   
   
   
 
 

Modified: kylin/site/docs21/howto/howto_build_cube_with_restapi.html
URL: 

svn commit: r1806793 [4/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.

[lidong]

Modified: kylin/site/docs21/install/hadoop_env.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs21/install/hadoop_env.html?rev=1806793=1806792=1806793=diff
==
--- kylin/site/docs21/install/hadoop_env.html (original)
+++ kylin/site/docs21/install/hadoop_env.html Thu Aug 31 14:01:54 2017
@@ -335,11 +335,7 @@
 
   
 
-  
-
-  
-
-  FAQ  
+  Technical Concepts   
   
   
 
 
@@ -359,7 +355,11 @@
 
   
 
-  Events and 
Conferences  
+  
+
+  
+
+  Terminology  
   
 
 
@@ -375,7 +375,13 @@
 
   
 
-  Community Best 
Practices  
+  
+
+  
+
+  
+
+  FAQ  
   
 
 
@@ -395,11 +401,7 @@
 
   
 
-  
-
-  
-
-  Terminology  
+  Events and 
Conferences  
   
 
 
@@ -415,9 +417,7 @@
 
   
 
-  
-
-  Technical Concepts   
   
+  Community Best 
Practices  
   
 
 
@@ -678,6 +678,8 @@
   
 
   
+
+  
 
 
 
@@ -1054,7 +1056,53 @@
 
   
 
-  Kylin Cube Creation  

+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Web Interface  
   
 
 
@@ -1132,11 +1180,7 @@
 
   
 
-  
-
-  
-
-  Kylin Cube Build and Job 
Monitoring  
+  Cube Wizard  
   
 
 
@@ -1218,11 +1262,7 @@
 
   
 
-  
-
-  
-
-  Build Cube with Spark 
(beta)  
+  Cube Build and Job 
Monitoring  
   
 
 
@@ -1296,7 +1336,7 @@
 
   
 
-  Kylin Cube Permission  
+  Cube Permission  
   
 
 
@@ -1382,6 +1422,54 @@
 
   
 
+  Build Cube with Spark 
(beta)  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
   
 
   
@@ -1418,11 +1506,29 @@
 
   
 
-  Kylin Web Interface  
+  Cube Build Tuning
  
   
 
 
 
+
+
+
+
+
+  
+
+
+Integration
+
+   
+
+
+
+
+
+
+
   
   
 
@@ -1520,9 +1626,21 @@
 
   
 
+  Kylin ODBC Driver  
+  
+
+
+
+  
+  
+
   
 
   
+
+
+
+  
 
   
 
@@ -1532,7 +1650,15 @@
 
   
 
-  Tableau 8  
+  
+
+  
+
+  
+
+  
+
+  Kylin JDBC Driver  
   
 
 
@@ -1648,9 +1774,7 @@
 
   
 
-  
-
-  Tableau 9  
+  Tableau 8  
   
 
 
@@ -1754,7 +1878,23 @@
 
   
 
-  MS Excel and Power BI  
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Tableau 9  
   
 
 
@@ -1854,7 +1994,13 @@
 
   
 
-  Kylin ODBC Driver  
+  
+
+  
+
+  
+
+  MS Excel and Power BI  
   
 
 
@@ -1944,7 +2090,15 @@
 
   
 
-  Connect from Apache Flink  
+  
+
+  
+
+  
+
+  
+
+  MicroStrategy  
   
 
 
@@ -2054,7 +2208,9 @@
 
   
 
-  Connect from SQuirreL  
+  
+
+  SQuirreL  
   
 
 
@@ -2138,7 +2294,13 @@
 
   
 
-  Cube Build Tuning Step by 
Step  
+  
+
+  
+
+  
+
+  Apache Flink  
   
 
 
@@ -2230,7 +2392,7 @@
 
   
 
-  Connect from Apache Hue  
+  Apache Hue  
   
 
 
@@ -2276,24 +2438,6 @@
 
   
 
-  Build Cube with RESTful 
API  
-  
-
-
-
-  
-  
-
-  
-
-  
-
-
-
-  
-
-  
-
   
 
   
@@ -2314,19 +2458,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API in 
Javascript  
+  Use RESTful API  
   
 
 
@@ -2354,27 +2486,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API  
+  Build Cube with API  

   
 
 
@@ -2504,7 +2616,7 @@
 
   
 
-  Cleanup Storage (HDFS & 
HBase)  
+  Cleanup Storage  
   
 
 
@@ -2536,7 +2648,7 @@
 
   
 
-  Use JDBC Driver  
+  Kylin JDBC Driver  
   
 
 
@@ -2612,7 +2724,7 @@
 
   
 
-  Enable Security with LDAP and 
SSO  
+  Secure with LDAP and 
SSO  
   
 
 
@@ -2656,7 +2768,7 @@
 
   
 
-  Use Beeline for Hive 
Commands  
+  Use Beeline for Hive 
 
   
 
 
@@ -2696,7 +2808,7 @@
 
   
 
-  How to Update HBase 
Coprocessor  
+  Update Coprocessor   
   
   
 
 

Modified: kylin/site/docs21/install/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs21/install/index.html?rev=1806793=1806792=1806793=diff

svn commit: r1806793 [7/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.

[lidong]

Modified: kylin/site/docs21/tutorial/tableau_91.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs21/tutorial/tableau_91.html?rev=1806793=1806792=1806793=diff
==
--- kylin/site/docs21/tutorial/tableau_91.html (original)
+++ kylin/site/docs21/tutorial/tableau_91.html Thu Aug 31 14:01:54 2017
@@ -335,11 +335,7 @@
 
   
 
-  
-
-  
-
-  FAQ  
+  Technical Concepts   
   
   
 
 
@@ -359,7 +355,11 @@
 
   
 
-  Events and 
Conferences  
+  
+
+  
+
+  Terminology  
   
 
 
@@ -375,7 +375,13 @@
 
   
 
-  Community Best 
Practices  
+  
+
+  
+
+  
+
+  FAQ  
   
 
 
@@ -395,11 +401,7 @@
 
   
 
-  
-
-  
-
-  Terminology  
+  Events and 
Conferences  
   
 
 
@@ -415,9 +417,7 @@
 
   
 
-  
-
-  Technical Concepts   
   
+  Community Best 
Practices  
   
 
 
@@ -678,6 +678,8 @@
   
 
   
+
+  
 
 
 
@@ -1054,7 +1056,53 @@
 
   
 
-  Kylin Cube Creation  

+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Web Interface  
   
 
 
@@ -1132,11 +1180,7 @@
 
   
 
-  
-
-  
-
-  Kylin Cube Build and Job 
Monitoring  
+  Cube Wizard  
   
 
 
@@ -1218,11 +1262,7 @@
 
   
 
-  
-
-  
-
-  Build Cube with Spark 
(beta)  
+  Cube Build and Job 
Monitoring  
   
 
 
@@ -1296,7 +1336,7 @@
 
   
 
-  Kylin Cube Permission  
+  Cube Permission  
   
 
 
@@ -1382,6 +1422,54 @@
 
   
 
+  Build Cube with Spark 
(beta)  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
   
 
   
@@ -1418,11 +1506,29 @@
 
   
 
-  Kylin Web Interface  
+  Cube Build Tuning
  
   
 
 
 
+
+
+
+
+
+  
+
+
+Integration
+
+   
+
+
+
+
+
+
+
   
   
 
@@ -1520,9 +1626,21 @@
 
   
 
+  Kylin ODBC Driver  
+  
+
+
+
+  
+  
+
   
 
   
+
+
+
+  
 
   
 
@@ -1532,7 +1650,15 @@
 
   
 
-  Tableau 8  
+  
+
+  
+
+  
+
+  
+
+  Kylin JDBC Driver  
   
 
 
@@ -1648,9 +1774,7 @@
 
   
 
-  
-
-  Tableau 9  
+  Tableau 8  
   
 
 
@@ -1754,7 +1878,23 @@
 
   
 
-  MS Excel and Power BI  
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Tableau 9  
   
 
 
@@ -1854,7 +1994,13 @@
 
   
 
-  Kylin ODBC Driver  
+  
+
+  
+
+  
+
+  MS Excel and Power BI  
   
 
 
@@ -1944,7 +2090,15 @@
 
   
 
-  Connect from Apache Flink  
+  
+
+  
+
+  
+
+  
+
+  MicroStrategy  
   
 
 
@@ -2054,7 +2208,9 @@
 
   
 
-  Connect from SQuirreL  
+  
+
+  SQuirreL  
   
 
 
@@ -2138,7 +2294,13 @@
 
   
 
-  Cube Build Tuning Step by 
Step  
+  
+
+  
+
+  
+
+  Apache Flink  
   
 
 
@@ -2230,7 +2392,7 @@
 
   
 
-  Connect from Apache Hue  
+  Apache Hue  
   
 
 
@@ -2276,24 +2438,6 @@
 
   
 
-  Build Cube with RESTful 
API  
-  
-
-
-
-  
-  
-
-  
-
-  
-
-
-
-  
-
-  
-
   
 
   
@@ -2314,19 +2458,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API in 
Javascript  
+  Use RESTful API  
   
 
 
@@ -2354,27 +2486,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API  
+  Build Cube with API  

   
 
 
@@ -2504,7 +2616,7 @@
 
   
 
-  Cleanup Storage (HDFS & 
HBase)  
+  Cleanup Storage  
   
 
 
@@ -2536,7 +2648,7 @@
 
   
 
-  Use JDBC Driver  
+  Kylin JDBC Driver  
   
 
 
@@ -2612,7 +2724,7 @@
 
   
 
-  Enable Security with LDAP and 
SSO  
+  Secure with LDAP and 
SSO  
   
 
 
@@ -2656,7 +2768,7 @@
 
   
 
-  Use Beeline for Hive 
Commands  
+  Use Beeline for Hive 
 
   
 
 
@@ -2696,7 +2808,7 @@
 
   
 
-  How to Update HBase 
Coprocessor  
+  Update Coprocessor   
   
   
 
 

Modified: kylin/site/docs21/tutorial/web.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs21/tutorial/web.html?rev=1806793=1806792=1806793=diff

svn commit: r1806793 [1/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.

[lidong]

Author: lidong
Date: Thu Aug 31 14:01:54 2017
New Revision: 1806793

URL: http://svn.apache.org/viewvc?rev=1806793=rev
Log:
Update doc layout

Added:
kylin/site/docs21/tutorial/microstrategy.html
kylin/site/images/tutorial/2.1/MicroStrategy/
kylin/site/images/tutorial/2.1/MicroStrategy/0.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/1.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/10.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/2.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/3.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/4.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/5.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/6.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/7.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/8.png   (with props)
kylin/site/images/tutorial/2.1/MicroStrategy/9.png   (with props)
Modified:
kylin/site/blog/index.html
kylin/site/cn/docs21/index.html
kylin/site/cn/docs21/install/index.html
kylin/site/cn/docs21/install/manual_install_guide.html
kylin/site/cn/docs21/tutorial/acl.html
kylin/site/cn/docs21/tutorial/create_cube.html
kylin/site/cn/docs21/tutorial/cube_build_job.html
kylin/site/cn/docs21/tutorial/kylin_client_tool.html
kylin/site/cn/docs21/tutorial/odbc.html
kylin/site/cn/docs21/tutorial/powerbi.html
kylin/site/cn/docs21/tutorial/query_pushdown.html
kylin/site/cn/docs21/tutorial/tableau.html
kylin/site/cn/docs21/tutorial/tableau_91.html
kylin/site/cn/docs21/tutorial/web.html
kylin/site/docs21/gettingstarted/best_practices.html
kylin/site/docs21/gettingstarted/concepts.html
kylin/site/docs21/gettingstarted/events.html
kylin/site/docs21/gettingstarted/faq.html
kylin/site/docs21/gettingstarted/terminology.html
kylin/site/docs21/howto/howto_backup_metadata.html
kylin/site/docs21/howto/howto_build_cube_with_restapi.html
kylin/site/docs21/howto/howto_cleanup_storage.html
kylin/site/docs21/howto/howto_jdbc.html
kylin/site/docs21/howto/howto_ldap_and_sso.html
kylin/site/docs21/howto/howto_optimize_build.html
kylin/site/docs21/howto/howto_optimize_cubes.html
kylin/site/docs21/howto/howto_update_coprocessor.html
kylin/site/docs21/howto/howto_upgrade.html
kylin/site/docs21/howto/howto_use_beeline.html
kylin/site/docs21/howto/howto_use_distributed_scheduler.html
kylin/site/docs21/howto/howto_use_restapi.html
kylin/site/docs21/howto/howto_use_restapi_in_js.html
kylin/site/docs21/index.html
kylin/site/docs21/install/advance_settings.html
kylin/site/docs21/install/hadoop_env.html
kylin/site/docs21/install/index.html
kylin/site/docs21/install/kylin_cluster.html
kylin/site/docs21/install/kylin_docker.html
kylin/site/docs21/release_notes.html
kylin/site/docs21/tutorial/acl.html
kylin/site/docs21/tutorial/create_cube.html
kylin/site/docs21/tutorial/cube_build_job.html
kylin/site/docs21/tutorial/cube_build_performance.html
kylin/site/docs21/tutorial/cube_spark.html
kylin/site/docs21/tutorial/cube_streaming.html
kylin/site/docs21/tutorial/flink.html
kylin/site/docs21/tutorial/hue.html
kylin/site/docs21/tutorial/kylin_sample.html
kylin/site/docs21/tutorial/odbc.html
kylin/site/docs21/tutorial/powerbi.html
kylin/site/docs21/tutorial/query_pushdown.html
kylin/site/docs21/tutorial/squirrel.html
kylin/site/docs21/tutorial/tableau.html
kylin/site/docs21/tutorial/tableau_91.html
kylin/site/docs21/tutorial/web.html
kylin/site/feed.xml

Modified: kylin/site/blog/index.html
URL: 
http://svn.apache.org/viewvc/kylin/site/blog/index.html?rev=1806793=1806792=1806793=diff
==
--- kylin/site/blog/index.html (original)
+++ kylin/site/blog/index.html Thu Aug 31 14:01:54 2017
@@ -301,13 +301,13 @@
 
 
 
-  Apache 
Kylin v1.5.1 Release Announcementposted: Apr 12, 2016
+  Apache Kylin v1.5.1 
正式发布posted: Apr 12, 
2016
 
   
 
 
 
-  Apache Kylin v1.5.1 
正式发布posted: Apr 12, 
2016
+  Apache 
Kylin v1.5.1 Release Announcementposted: Apr 12, 2016
 
   
 
@@ -379,13 +379,13 @@
 
 
 
-  Apache Kylin v1.1 (incubating) 
Release Announcementposted: Oct 
25, 2015
+  Apache Kylin v1.1 
(incubating) 正式发布posted: 
Oct 25, 2015
 
   
 
 
 
-  Apache Kylin v1.1 
(incubating) 正式发布posted: 
Oct 25, 2015
+  Apache Kylin v1.1 (incubating) 
Release Announcementposted: Oct 
25, 2015
 
   
 

Modified: kylin/site/cn/docs21/index.html
URL: 

svn commit: r1806793 [3/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.

[lidong]

Modified: kylin/site/docs21/howto/howto_upgrade.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs21/howto/howto_upgrade.html?rev=1806793=1806792=1806793=diff
==
--- kylin/site/docs21/howto/howto_upgrade.html (original)
+++ kylin/site/docs21/howto/howto_upgrade.html Thu Aug 31 14:01:54 2017
@@ -335,11 +335,7 @@
 
   
 
-  
-
-  
-
-  FAQ  
+  Technical Concepts   
   
   
 
 
@@ -359,7 +355,11 @@
 
   
 
-  Events and 
Conferences  
+  
+
+  
+
+  Terminology  
   
 
 
@@ -375,7 +375,13 @@
 
   
 
-  Community Best 
Practices  
+  
+
+  
+
+  
+
+  FAQ  
   
 
 
@@ -395,11 +401,7 @@
 
   
 
-  
-
-  
-
-  Terminology  
+  Events and 
Conferences  
   
 
 
@@ -415,9 +417,7 @@
 
   
 
-  
-
-  Technical Concepts   
   
+  Community Best 
Practices  
   
 
 
@@ -678,6 +678,8 @@
   
 
   
+
+  
 
 
 
@@ -1054,7 +1056,53 @@
 
   
 
-  Kylin Cube Creation  

+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Web Interface  
   
 
 
@@ -1132,11 +1180,7 @@
 
   
 
-  
-
-  
-
-  Kylin Cube Build and Job 
Monitoring  
+  Cube Wizard  
   
 
 
@@ -1218,11 +1262,7 @@
 
   
 
-  
-
-  
-
-  Build Cube with Spark 
(beta)  
+  Cube Build and Job 
Monitoring  
   
 
 
@@ -1296,7 +1336,7 @@
 
   
 
-  Kylin Cube Permission  
+  Cube Permission  
   
 
 
@@ -1382,6 +1422,54 @@
 
   
 
+  Build Cube with Spark 
(beta)  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
   
 
   
@@ -1418,11 +1506,29 @@
 
   
 
-  Kylin Web Interface  
+  Cube Build Tuning
  
   
 
 
 
+
+
+
+
+
+  
+
+
+Integration
+
+   
+
+
+
+
+
+
+
   
   
 
@@ -1520,9 +1626,21 @@
 
   
 
+  Kylin ODBC Driver  
+  
+
+
+
+  
+  
+
   
 
   
+
+
+
+  
 
   
 
@@ -1532,7 +1650,15 @@
 
   
 
-  Tableau 8  
+  
+
+  
+
+  
+
+  
+
+  Kylin JDBC Driver  
   
 
 
@@ -1648,9 +1774,7 @@
 
   
 
-  
-
-  Tableau 9  
+  Tableau 8  
   
 
 
@@ -1754,7 +1878,23 @@
 
   
 
-  MS Excel and Power BI  
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Tableau 9  
   
 
 
@@ -1854,7 +1994,13 @@
 
   
 
-  Kylin ODBC Driver  
+  
+
+  
+
+  
+
+  MS Excel and Power BI  
   
 
 
@@ -1944,7 +2090,15 @@
 
   
 
-  Connect from Apache Flink  
+  
+
+  
+
+  
+
+  
+
+  MicroStrategy  
   
 
 
@@ -2054,7 +2208,9 @@
 
   
 
-  Connect from SQuirreL  
+  
+
+  SQuirreL  
   
 
 
@@ -2138,7 +2294,13 @@
 
   
 
-  Cube Build Tuning Step by 
Step  
+  
+
+  
+
+  
+
+  Apache Flink  
   
 
 
@@ -2230,7 +2392,7 @@
 
   
 
-  Connect from Apache Hue  
+  Apache Hue  
   
 
 
@@ -2276,24 +2438,6 @@
 
   
 
-  Build Cube with RESTful 
API  
-  
-
-
-
-  
-  
-
-  
-
-  
-
-
-
-  
-
-  
-
   
 
   
@@ -2314,19 +2458,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API in 
Javascript  
+  Use RESTful API  
   
 
 
@@ -2354,27 +2486,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API  
+  Build Cube with API  

   
 
 
@@ -2504,7 +2616,7 @@
 
   
 
-  Cleanup Storage (HDFS & 
HBase)  
+  Cleanup Storage  
   
 
 
@@ -2536,7 +2648,7 @@
 
   
 
-  Use JDBC Driver  
+  Kylin JDBC Driver  
   
 
 
@@ -2612,7 +2724,7 @@
 
   
 
-  Enable Security with LDAP and 
SSO  
+  Secure with LDAP and 
SSO  
   
 
 
@@ -2656,7 +2768,7 @@
 
   
 
-  Use Beeline for Hive 
Commands  
+  Use Beeline for Hive 
 
   
 
 
@@ -2696,7 +2808,7 @@
 
   
 
-  How to Update HBase 
Coprocessor  
+  Update Coprocessor   
   
   
 
 

Modified: kylin/site/docs21/howto/howto_use_beeline.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs21/howto/howto_use_beeline.html?rev=1806793=1806792=1806793=diff

svn commit: r1806793 [5/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.

[lidong]

Modified: kylin/site/docs21/tutorial/cube_build_performance.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs21/tutorial/cube_build_performance.html?rev=1806793=1806792=1806793=diff
==
--- kylin/site/docs21/tutorial/cube_build_performance.html (original)
+++ kylin/site/docs21/tutorial/cube_build_performance.html Thu Aug 31 14:01:54 
2017
@@ -41,7 +41,7 @@
   
   
 
-  Apache Kylin | Cube Build Tuning Step by Step
+  Apache Kylin | Cube Build Tuning
   
   
   
@@ -335,11 +335,7 @@
 
   
 
-  
-
-  
-
-  FAQ  
+  Technical Concepts   
   
   
 
 
@@ -359,7 +355,11 @@
 
   
 
-  Events and 
Conferences  
+  
+
+  
+
+  Terminology  
   
 
 
@@ -375,7 +375,13 @@
 
   
 
-  Community Best 
Practices  
+  
+
+  
+
+  
+
+  FAQ  
   
 
 
@@ -395,11 +401,7 @@
 
   
 
-  
-
-  
-
-  Terminology  
+  Events and 
Conferences  
   
 
 
@@ -415,9 +417,7 @@
 
   
 
-  
-
-  Technical Concepts   
   
+  Community Best 
Practices  
   
 
 
@@ -678,6 +678,8 @@
   
 
   
+
+  
 
 
 
@@ -1054,7 +1056,53 @@
 
   
 
-  Kylin Cube Creation  

+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Web Interface  
   
 
 
@@ -1132,11 +1180,7 @@
 
   
 
-  
-
-  
-
-  Kylin Cube Build and Job 
Monitoring  
+  Cube Wizard  
   
 
 
@@ -1218,11 +1262,7 @@
 
   
 
-  
-
-  
-
-  Build Cube with Spark 
(beta)  
+  Cube Build and Job 
Monitoring  
   
 
 
@@ -1296,7 +1336,7 @@
 
   
 
-  Kylin Cube Permission  
+  Cube Permission  
   
 
 
@@ -1382,6 +1422,54 @@
 
   
 
+  Build Cube with Spark 
(beta)  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
   
 
   
@@ -1418,11 +1506,29 @@
 
   
 
-  Kylin Web Interface  
+  Cube Build Tuning
  
   
 
 
 
+
+
+
+
+
+  
+
+
+Integration
+
+   
+
+
+
+
+
+
+
   
   
 
@@ -1520,9 +1626,21 @@
 
   
 
+  Kylin ODBC Driver  
+  
+
+
+
+  
+  
+
   
 
   
+
+
+
+  
 
   
 
@@ -1532,7 +1650,15 @@
 
   
 
-  Tableau 8  
+  
+
+  
+
+  
+
+  
+
+  Kylin JDBC Driver  
   
 
 
@@ -1648,9 +1774,7 @@
 
   
 
-  
-
-  Tableau 9  
+  Tableau 8  
   
 
 
@@ -1754,7 +1878,23 @@
 
   
 
-  MS Excel and Power BI  
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Tableau 9  
   
 
 
@@ -1854,7 +1994,13 @@
 
   
 
-  Kylin ODBC Driver  
+  
+
+  
+
+  
+
+  MS Excel and Power BI  
   
 
 
@@ -1944,7 +2090,15 @@
 
   
 
-  Connect from Apache Flink  
+  
+
+  
+
+  
+
+  
+
+  MicroStrategy  
   
 
 
@@ -2054,7 +2208,9 @@
 
   
 
-  Connect from SQuirreL  
+  
+
+  SQuirreL  
   
 
 
@@ -2138,7 +2294,13 @@
 
   
 
-  Cube Build Tuning Step by 
Step  
+  
+
+  
+
+  
+
+  Apache Flink  
   
 
 
@@ -2230,7 +2392,7 @@
 
   
 
-  Connect from Apache Hue  
+  Apache Hue  
   
 
 
@@ -2276,24 +2438,6 @@
 
   
 
-  Build Cube with RESTful 
API  
-  
-
-
-
-  
-  
-
-  
-
-  
-
-
-
-  
-
-  
-
   
 
   
@@ -2314,19 +2458,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API in 
Javascript  
+  Use RESTful API  
   
 
 
@@ -2354,27 +2486,7 @@
 
   
 
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  
-
-  Use RESTful API  
+  Build Cube with API  

   
 
 
@@ -2504,7 +2616,7 @@
 
   
 
-  Cleanup Storage (HDFS & 
HBase)  
+  Cleanup Storage  
   
 
 
@@ -2536,7 +2648,7 @@
 
   
 
-  Use JDBC Driver  
+  Kylin JDBC Driver  
   
 
 
@@ -2612,7 +2724,7 @@
 
   
 
-  Enable Security with LDAP and 
SSO  
+  Secure with LDAP and 
SSO  
   
 
 
@@ -2656,7 +2768,7 @@
 
   
 
-  Use Beeline for Hive 
Commands  
+  Use Beeline for Hive 
 
   
 
 
@@ -2696,7 +2808,7 @@
 
   
 
-  How to Update HBase 
Coprocessor  
+  Update Coprocessor   
   
   
 
 
@@ -2716,7 +2828,7 

svn commit: r1806793 [6/7] - in /kylin/site: ./ blog/ cn/docs21/ cn/docs21/install/ cn/docs21/tutorial/ docs21/ docs21/gettingstarted/ docs21/howto/ docs21/install/ docs21/tutorial/ images/tutorial/2.

[lidong]

Added: kylin/site/docs21/tutorial/microstrategy.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs21/tutorial/microstrategy.html?rev=1806793=auto
==
--- kylin/site/docs21/tutorial/microstrategy.html (added)
+++ kylin/site/docs21/tutorial/microstrategy.html Thu Aug 31 14:01:54 2017
@@ -0,0 +1,2976 @@
+
+
+
+
+   
+
+
+  
+  
+  
+
+  Apache Kylin | MicroStrategy
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+  
+  
+
+  http://kylin.apache.org/docs21/tutorial/microstrategy.html;>
+  http://kylin.apache.org/feed.xml; />
+
+
+
+  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new 
Date();a=s.createElement(o),
+  
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+  //oringal tracker for kylin.io
+  ga('create', 'UA-55534813-1', 'auto');
+  //new tracker for kylin.apache.org
+  ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'});
+
+  ga('send', 'pageview');
+  ga('toplevel.send', 'pageview');
+
+
+
+
+ 
+ 
+
+
+   
+   
+
+
+  
+  
+  Apache Kylin™ Extreme OLAP Engine for Big 
Data 
+
+
+  http://apache.org/foundation/contributing.html; title="Support 
Apache" style="margin-left: 150px;">
+  https://www.apache.org/images/SupportApache-small.png; 
style="height: 150px; width: 150px;">
+  
+  
+  
+  
+
+  
+  
+  
+
+
+
+  
+Toggle navigation
+
+
+
+  
+ 
+
+
+
+
+  
+ Home
+  Docs
+  Download
+  Community
+  Development
+  Blog
+  中文版  
+  https://twitter.com/apachekylin; target="_blank" 
class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" >
+  https://github.com/apache/kylin; target="_blank" 
class="fa fa-github-alt fa-lg" title="Github: apache/kylin" >  
+  https://www.facebook.com/kylinio; target="_blank" 
class="fa fa-facebook fa-lg" title="Facebook: kylin.io" >   
+
+
+  
+
+ 
+
+   
+   
+   
+   
+
+
+
+
+Getting Started
+
+   
+
+
+
+
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Overview  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Release Notes  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  Technical Concepts   
   
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Terminology  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  FAQ  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  Events and 
Conferences  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  Community Best 
Practices  
+  
+
+
+
+
+
+
+
+
+  
+
+
+Installation
+
+   
+
+
+
+
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Installation Guide  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  Hadoop Environment  
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+  

[3/4] kylin git commit: minor changes to MicroStrategy tutorial

[shaofengshi]

minor changes to MicroStrategy tutorial

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2291fcee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2291fcee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2291fcee

Branch: refs/heads/document
Commit: 2291fcee5b6f0b66c4782372b8ded976352125da
Parents: c8126da
Author: joannahe 
Authored: Thu Aug 31 17:26:39 2017 +0800
Committer: shaofengshi 
Committed: Thu Aug 31 17:29:12 2017 +0800

--
 website/_docs21/tutorial/microstrategy.md   |  22 +++
 website/images/tutorial/2.1/MicroStrategy/0.png | Bin 0 -> 207801 bytes
 2 files changed, 13 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2291fcee/website/_docs21/tutorial/microstrategy.md
--
diff --git a/website/_docs21/tutorial/microstrategy.md 
b/website/_docs21/tutorial/microstrategy.md
index 7c549eb..67ec4c3 100644
--- a/website/_docs21/tutorial/microstrategy.md
+++ b/website/_docs21/tutorial/microstrategy.md
@@ -1,31 +1,35 @@
 ---
 layout: docs21
-title:  MicroStrategy 10
+title:  Integration with MicroStrategy 10.X
 categories: tutorial
 permalink: /docs21/tutorial/microstrategy.html
 ---
 
-## Integration with MicroStrategy 10.X
-
 ### Install ODBC Driver
 
 Refer to this guide: [Kylin ODBC Driver Tutorial](./odbc.html).
-Please make sure to download and install Kylin ODBC Driver __v1.6__ or above. 
If you already installed ODBC Driver in your system, please uninstall it first. 
already installed ODBC Driver in your system, please uninstall it first.  
+Please make sure to download and install Kylin ODBC Driver __v1.6__ 64 bit or 
above. If you already installed ODBC Driver in your system, please uninstall it 
first. already installed ODBC Driver in your system, please uninstall it first. 
 
 
-### Connect to Kylin Server
+The Kylin ODBC driver needs to be installed in the machine or virtual 
environment where your Microstrategy Intelligenec Server is installed. 
 
-Connect Kylin using ODBC driver: open your MicroStrategy Developer and connect 
to the project source where your are going to connect Kylin data source using a 
user account with administrative privilege. 
+###Create Local DSN
 
-Once logged in, go to `Administration` -> `Configuration manager` -> `Database 
Instance`, create a new database instance with local ODBC connection that you 
created in the previous step. Under database connection type, please choose 
Generic DBMS.
+Open your window ODBC Data Source Administrator (64bit) and create a system 
DSN that point to your kylin instance. 
+
+![](/images/tutorial/2.1/MicroStrategy/0.png)
 
 ### Setting Database Instance
 
-Depending on your business scenario, you may need to create a new project and 
set Kylin database instance as your primary database instance or if there is an 
existing project, set Kylin database instance as one of your primary or 
non-primary database instance. You can achieve this by right click on your 
project, and go to `project configuration` -> `database instance`. 
+Connect Kylin using ODBC driver: open your MicroStrategy Developer and connect 
to the project source where your are going to connect Kylin data source using a 
user account with administrative privilege. 
+
+Once logged in, go to `Administration` -> `Configuration manager` -> `Database 
Instance`, create a new database instance with system DSN that you created in 
the previous step. Under database connection type, please choose Generic DBMS.
 
 ![](/images/tutorial/2.1/MicroStrategy/2.png)
 
 ![](/images/tutorial/2.1/MicroStrategy/1.png)
 
+Depending on your business scenario, you may need to create a new project and 
set Kylin database instance as your primary database instance or if there is an 
existing project, set Kylin database instance as one of your primary or 
non-primary database instance. You can achieve this by right click on your 
project, and go to `project configuration` -> `database instance`. 
+
 ### Import Logical Table
 
 Open up your project, go to `schema` -> `warehouse catalog` to import the 
tables your need. 
@@ -77,4 +81,4 @@ Now you can start creating reports with Kylin as data source.
3. On the top menu choose `Tools` -> `show Advanced Settings`.
4. Go to `select/insert` -> `date format`.
5. Change the date format to follow date format in Kylin, for example 
'-mm-dd'.
-   6.  Restart MicroStrategy Intelligence Server so that change can be 
effective. 
\ No newline at end of file
+   6. Restart MicroStrategy Intelligence Server so that change can be 
effective. 
\ No newline at end of file

[4/4] kylin git commit: Update doc layout

[shaofengshi]

Update doc layout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1af8222
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1af8222
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1af8222

Branch: refs/heads/document
Commit: c1af8222f1b7d8196a2065e1cf3adfc638730cb8
Parents: 2291fce
Author: shaofengshi 
Authored: Thu Aug 31 18:57:47 2017 +0800
Committer: shaofengshi 
Committed: Thu Aug 31 18:57:47 2017 +0800

--
 website/_data/docs21.yml| 22 ++
 .../howto/howto_build_cube_with_restapi.md  |  2 +-
 website/_docs21/howto/howto_cleanup_storage.md  |  2 +-
 website/_docs21/howto/howto_jdbc.md |  2 +-
 website/_docs21/howto/howto_ldap_and_sso.md |  2 +-
 .../_docs21/howto/howto_update_coprocessor.md   |  2 +-
 website/_docs21/howto/howto_use_beeline.md  |  2 +-
 website/_docs21/howto/howto_use_restapi.md  | 44 
 website/_docs21/index.md| 16 +++
 website/_docs21/tutorial/acl.md |  2 +-
 website/_docs21/tutorial/create_cube.md |  2 +-
 website/_docs21/tutorial/cube_build_job.md  |  2 +-
 .../_docs21/tutorial/cube_build_performance.md  |  2 +-
 website/_docs21/tutorial/flink.md   |  2 +-
 website/_docs21/tutorial/hue.md |  2 +-
 website/_docs21/tutorial/microstrategy.md   |  2 +-
 website/_docs21/tutorial/squirrel.md|  2 +-
 website/_docs21/tutorial/web.md |  2 +-
 18 files changed, 80 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c1af8222/website/_data/docs21.yml
--
diff --git a/website/_data/docs21.yml b/website/_data/docs21.yml
index 3f3380f..90acd70 100644
--- a/website/_data/docs21.yml
+++ b/website/_data/docs21.yml
@@ -19,11 +19,11 @@
   docs:
   - index
   - release_notes
+  - gettingstarted/concepts
+  - gettingstarted/terminology
   - gettingstarted/faq
   - gettingstarted/events
   - gettingstarted/best_practices
-  - gettingstarted/terminology
-  - gettingstarted/concepts
 
 - title: Installation
   docs:
@@ -37,25 +37,29 @@
 - title: Tutorial
   docs:
   - tutorial/kylin_sample
+  - tutorial/web
   - tutorial/create_cube
   - tutorial/cube_build_job
-  - tutorial/cube_spark
   - tutorial/acl
-  - tutorial/web
+  - tutorial/cube_spark
+  - tutorial/cube_build_performance
+
+- title: Integration
+  docs:
+  - tutorial/odbc
+  - howto/howto_jdbc
   - tutorial/tableau
   - tutorial/tableau_91
   - tutorial/powerbi
-  - tutorial/odbc
-  - tutorial/flink
+  - tutorial/microstrategy
   - tutorial/squirrel
-  - tutorial/cube_build_performance
+  - tutorial/flink
   - tutorial/hue
 
 - title: How To
   docs:
-  - howto/howto_build_cube_with_restapi
-  - howto/howto_use_restapi_in_js
   - howto/howto_use_restapi
+  - howto/howto_build_cube_with_restapi
   - howto/howto_optimize_cubes
   - howto/howto_optimize_build
   - howto/howto_backup_metadata

http://git-wip-us.apache.org/repos/asf/kylin/blob/c1af8222/website/_docs21/howto/howto_build_cube_with_restapi.md
--
diff --git a/website/_docs21/howto/howto_build_cube_with_restapi.md 
b/website/_docs21/howto/howto_build_cube_with_restapi.md
index 9092afe..8808b7b 100644
--- a/website/_docs21/howto/howto_build_cube_with_restapi.md
+++ b/website/_docs21/howto/howto_build_cube_with_restapi.md
@@ -1,6 +1,6 @@
 ---
 layout: docs21
-title:  Build Cube with RESTful API
+title:  Build Cube with API
 categories: howto
 permalink: /docs21/howto/howto_build_cube_with_restapi.html
 ---

http://git-wip-us.apache.org/repos/asf/kylin/blob/c1af8222/website/_docs21/howto/howto_cleanup_storage.md
--
diff --git a/website/_docs21/howto/howto_cleanup_storage.md 
b/website/_docs21/howto/howto_cleanup_storage.md
index 323e138..f488040 100644
--- a/website/_docs21/howto/howto_cleanup_storage.md
+++ b/website/_docs21/howto/howto_cleanup_storage.md
@@ -1,6 +1,6 @@
 ---
 layout: docs21
-title:  Cleanup Storage (HDFS & HBase)
+title:  Cleanup Storage
 categories: howto
 permalink: /docs21/howto/howto_cleanup_storage.html
 ---

http://git-wip-us.apache.org/repos/asf/kylin/blob/c1af8222/website/_docs21/howto/howto_jdbc.md
--
diff --git a/website/_docs21/howto/howto_jdbc.md 
b/website/_docs21/howto/howto_jdbc.md
index f2fd6f0..4885a84 100644
--- a/website/_docs21/howto/howto_jdbc.md
+++ b/website/_docs21/howto/howto_jdbc.md
@@ -1,6 +1,6 @@
 ---
 layout: docs21
-title:  Use JDBC Driver
+title:  Kylin JDBC Driver
 categories: howto

[2/4] kylin git commit: minor update

[shaofengshi]

minor update

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8126dac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8126dac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8126dac

Branch: refs/heads/document
Commit: c8126dac9e7a8868b9aa31fafb30a20b4b09286c
Parents: 6555f6d
Author: joannahe 
Authored: Thu Aug 31 17:03:53 2017 +0800
Committer: shaofengshi 
Committed: Thu Aug 31 17:11:02 2017 +0800

--
 website/_docs21/tutorial/microstrategy.md | 18 +-
 1 file changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/c8126dac/website/_docs21/tutorial/microstrategy.md
--
diff --git a/website/_docs21/tutorial/microstrategy.md 
b/website/_docs21/tutorial/microstrategy.md
index 8a7cc45..7c549eb 100644
--- a/website/_docs21/tutorial/microstrategy.md
+++ b/website/_docs21/tutorial/microstrategy.md
@@ -5,7 +5,7 @@ categories: tutorial
 permalink: /docs21/tutorial/microstrategy.html
 ---
 
-## Integration from MicroStrategy 10.X
+## Integration with MicroStrategy 10.X
 
 ### Install ODBC Driver
 
@@ -18,15 +18,15 @@ Connect Kylin using ODBC driver: open your MicroStrategy 
Developer and connect t
 
 Once logged in, go to `Administration` -> `Configuration manager` -> `Database 
Instance`, create a new database instance with local ODBC connection that you 
created in the previous step. Under database connection type, please choose 
Generic DBMS.
 
-![](/images/tutorial/2.1/MicroStrategy/1.png)
+### Setting Database Instance
 
-### Setting Database instance
+Depending on your business scenario, you may need to create a new project and 
set Kylin database instance as your primary database instance or if there is an 
existing project, set Kylin database instance as one of your primary or 
non-primary database instance. You can achieve this by right click on your 
project, and go to `project configuration` -> `database instance`. 
 
-Depending on your business scenario, you may need to create a new project and 
set kylin database instance as your primary database instance or if there is an 
existing project, set kylin database instance as one of your primary or 
non-primary database instance. You can achieve this by right click on your 
project, and go to `project configuration` -> `database instance`. 
+![](/images/tutorial/2.1/MicroStrategy/2.png)
 
 ![](/images/tutorial/2.1/MicroStrategy/1.png)
 
-### Import logical table
+### Import Logical Table
 
 Open up your project, go to `schema` -> `warehouse catalog` to import the 
tables your need. 
 
@@ -46,13 +46,13 @@ Create Attribute, Facts and Metric objects
 
 ### Create a Simple Report
 
-Now you can start creating reports with Kylin as Data source.
+Now you can start creating reports with Kylin as data source.
 
 ![](/images/tutorial/2.1/MicroStrategy/9.png)
 
 ![](/images/tutorial/2.1/MicroStrategy/10.png)
 
-### Best Practice for connecting MicroStrategy to Kylin datasource
+### Best Practice for Connecting MicroStrategy to Kylin Data Source
 
 1. Kylin does not work with multiple SQL passes at the moment, so it is 
recommended to set up your report intermediate table type as derived, you can 
change this setting at report level using `Data`-> `VLDB property`-> `Tables`-> 
`Intermediate Table Type`
 
@@ -64,7 +64,7 @@ Now you can start creating reports with Kylin as Data source.
 
​ Reports with custom groups
 
-3. Dimension named with kylin keywords will cause sql to error out. You may 
find kylin keywords here, it is recommended to avoid naming the column name as 
kylin keywords, especially when you use MicroStrategy as the front-end BI tool, 
as far as I am concern there is no setting MicroStrategy that can escape the 
keyword.  
[https://calcite.apache.org/docs/reference.html#keywords](https://calcite.apache.org/docs/reference.html#keywords)
+3. Dimension named with Kylin keywords will cause sql to error out. You may 
find Kylin keywords here, it is recommended to avoid naming the column name as 
Kylin keywords, especially when you use MicroStrategy as the front-end BI tool, 
as far as we know there is no setting in MicroStrategy that can escape the 
keyword.  
[https://calcite.apache.org/docs/reference.html#keywords](https://calcite.apache.org/docs/reference.html#keywords)
 
 4. If underlying Kylin data model has left join from fact table to lookup 
table, In order for Microstrategy to also generate the same left join in sql, 
please follow below MicroStrategy TN to modify VLDB property:
 
@@ -76,5 +76,5 @@ Now you can start creating reports with Kylin as Data source.
2. Then right click on the database, 

[1/4] kylin git commit: add tutorial for MicroStrategy

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/document 08ab05ec4 -> c1af8222f


add tutorial for MicroStrategy

Signed-off-by: shaofengshi 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6555f6d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6555f6d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6555f6d2

Branch: refs/heads/document
Commit: 6555f6d2628f190cfd3442732875b83120e88312
Parents: 08ab05e
Author: joannahe 
Authored: Thu Aug 31 16:30:58 2017 +0800
Committer: shaofengshi 
Committed: Thu Aug 31 16:35:00 2017 +0800

--
 website/_docs21/tutorial/microstrategy.md   |  80 +++
 website/images/tutorial/2.1/MicroStrategy/1.png | Bin 0 -> 207356 bytes
 .../images/tutorial/2.1/MicroStrategy/10.png| Bin 0 -> 185351 bytes
 website/images/tutorial/2.1/MicroStrategy/2.png | Bin 0 -> 154312 bytes
 website/images/tutorial/2.1/MicroStrategy/3.png | Bin 0 -> 345509 bytes
 website/images/tutorial/2.1/MicroStrategy/4.png | Bin 0 -> 187026 bytes
 website/images/tutorial/2.1/MicroStrategy/5.png | Bin 0 -> 158694 bytes
 website/images/tutorial/2.1/MicroStrategy/6.png | Bin 0 -> 143233 bytes
 website/images/tutorial/2.1/MicroStrategy/7.png | Bin 0 -> 125824 bytes
 website/images/tutorial/2.1/MicroStrategy/8.png | Bin 0 -> 283974 bytes
 website/images/tutorial/2.1/MicroStrategy/9.png | Bin 0 -> 357212 bytes
 11 files changed, 80 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6555f6d2/website/_docs21/tutorial/microstrategy.md
--
diff --git a/website/_docs21/tutorial/microstrategy.md 
b/website/_docs21/tutorial/microstrategy.md
new file mode 100644
index 000..8a7cc45
--- /dev/null
+++ b/website/_docs21/tutorial/microstrategy.md
@@ -0,0 +1,80 @@
+---
+layout: docs21
+title:  MicroStrategy 10
+categories: tutorial
+permalink: /docs21/tutorial/microstrategy.html
+---
+
+## Integration from MicroStrategy 10.X
+
+### Install ODBC Driver
+
+Refer to this guide: [Kylin ODBC Driver Tutorial](./odbc.html).
+Please make sure to download and install Kylin ODBC Driver __v1.6__ or above. 
If you already installed ODBC Driver in your system, please uninstall it first. 
already installed ODBC Driver in your system, please uninstall it first.  
+
+### Connect to Kylin Server
+
+Connect Kylin using ODBC driver: open your MicroStrategy Developer and connect 
to the project source where your are going to connect Kylin data source using a 
user account with administrative privilege. 
+
+Once logged in, go to `Administration` -> `Configuration manager` -> `Database 
Instance`, create a new database instance with local ODBC connection that you 
created in the previous step. Under database connection type, please choose 
Generic DBMS.
+
+![](/images/tutorial/2.1/MicroStrategy/1.png)
+
+### Setting Database instance
+
+Depending on your business scenario, you may need to create a new project and 
set kylin database instance as your primary database instance or if there is an 
existing project, set kylin database instance as one of your primary or 
non-primary database instance. You can achieve this by right click on your 
project, and go to `project configuration` -> `database instance`. 
+
+![](/images/tutorial/2.1/MicroStrategy/1.png)
+
+### Import logical table
+
+Open up your project, go to `schema` -> `warehouse catalog` to import the 
tables your need. 
+
+![](/images/tutorial/2.1/MicroStrategy/4.png)
+
+### Building Schema and Public Objects
+
+Create Attribute, Facts and Metric objects
+
+![](/images/tutorial/2.1/MicroStrategy/5.png)
+
+![](/images/tutorial/2.1/MicroStrategy/6.png)
+
+![](/images/tutorial/2.1/MicroStrategy/7.png)
+
+![](/images/tutorial/2.1/MicroStrategy/8.png)
+
+### Create a Simple Report
+
+Now you can start creating reports with Kylin as Data source.
+
+![](/images/tutorial/2.1/MicroStrategy/9.png)
+
+![](/images/tutorial/2.1/MicroStrategy/10.png)
+
+### Best Practice for connecting MicroStrategy to Kylin datasource
+
+1. Kylin does not work with multiple SQL passes at the moment, so it is 
recommended to set up your report intermediate table type as derived, you can 
change this setting at report level using `Data`-> `VLDB property`-> `Tables`-> 
`Intermediate Table Type`
+
+2. Avoid using below functionality in MicroStrategy as it will generate 
multiple sql passes that can not be bypassed by VLDB property:
+
+   ​ Creation of datamarts
+
+   ​ Query partitioned tables
+
+   ​ Reports with custom groups
+
+3. Dimension named with kylin keywords will cause sql to error out. You may 
find kylin keywords here, it is recommended to avoid naming the column name as 
kylin keywords, especially when you use 

[3/4] kylin git commit: KYLIN-2814: fix connection may not be released when sync wipe cache

[billyliu]

KYLIN-2814: fix connection may not be released when sync wipe cache

Signed-off-by: Billy Liu 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c1aab10
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c1aab10
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c1aab10

Branch: refs/heads/KYLIN-2814
Commit: 5c1aab101f3d5a3fb0cdc4c85c1c3f60274979fc
Parents: 727920b
Author: lichao 
Authored: Fri Aug 25 14:31:17 2017 +0800
Committer: Billy Liu 
Committed: Thu Aug 31 15:38:31 2017 +0800

--
 .../java/org/apache/kylin/common/restclient/RestClient.java| 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1aab10/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java 
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 13490cb..efd0438 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -121,8 +121,9 @@ public class RestClient {
 String url = baseUrl + "/cache/" + entity + "/" + cacheKey + "/" + 
event;
 HttpPut request = new HttpPut(url);
 
+HttpResponse response = null;
 try {
-HttpResponse response = client.execute(request);
+response = client.execute(request);
 
 if (response.getStatusLine().getStatusCode() != 200) {
 String msg = EntityUtils.toString(response.getEntity());
@@ -131,6 +132,9 @@ public class RestClient {
 } catch (Exception ex) {
 throw new IOException(ex);
 } finally {
+if(response != null){
+EntityUtils.consume(response.getEntity());
+}
 request.releaseConnection();
 }
 }

[1/4] kylin git commit: KYLIN-2811, refine spark cubing [Forced Update!]

[billyliu]

Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2814 abd13082b -> 3f80488e3 (forced update)


KYLIN-2811, refine spark cubing

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1c234a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1c234a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1c234a9

Branch: refs/heads/KYLIN-2814
Commit: a1c234a9afbc5a30306f4275127649f980ab75bd
Parents: 4316cfd
Author: Cheng Wang 
Authored: Sun Aug 27 13:29:12 2017 +0800
Committer: Hongbin Ma 
Committed: Sun Aug 27 15:08:18 2017 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  50 +--
 .../metadata/model/ComputedColumnDesc.java  |  10 +-
 .../kylin/engine/mr/common/BatchConstants.java  |   1 +
 .../spark/SparkBatchCubingJobBuilder2.java  |   1 +
 .../kylin/engine/spark/SparkCubingByLayer.java  | 380 ---
 .../localmeta/cube_desc/ci_inner_join_cube.json |   2 +-
 .../kylin/rest/util/Log4jConfigListener.java|   1 +
 7 files changed, 267 insertions(+), 178 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/a1c234a9/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index a56e9b8..1d5e0ec 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -18,6 +18,15 @@
 
 package org.apache.kylin.common;
 
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.restclient.RestClient;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.OrderedProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
@@ -34,26 +43,15 @@ import java.util.Enumeration;
 import java.util.Map;
 import java.util.Properties;
 
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.restclient.RestClient;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.OrderedProperties;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
 /**
  */
 public class KylinConfig extends KylinConfigBase {
 private static final long serialVersionUID = 1L;
 private static final Logger logger = 
LoggerFactory.getLogger(KylinConfig.class);
 
-/** Kylin properties file name */
+/**
+ * Kylin properties file name
+ */
 public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties";
 public static final String KYLIN_CONF = "KYLIN_CONF";
 
@@ -62,7 +60,7 @@ public class KylinConfig extends KylinConfigBase {
 
 // thread-local instances, will override SYS_ENV_INSTANCE
 private static transient ThreadLocal THREAD_ENV_INSTANCE = 
new ThreadLocal<>();
-
+
 static {
 /*
  * Make Calcite to work with Unicode.
@@ -121,9 +119,6 @@ public class KylinConfig extends KylinConfigBase {
 }
 
 private static UriType decideUriType(String metaUri) {
-if (metaUri.indexOf("@hdfs") > 0) {
-return UriType.HDFS_FILE;
-}
 
 try {
 File file = new File(metaUri);
@@ -163,23 +158,6 @@ public class KylinConfig extends KylinConfigBase {
  */
 UriType uriType = decideUriType(uri);
 
-if (uriType == UriType.HDFS_FILE) {
-KylinConfig config;
-FileSystem fs;
-int cut = uri.indexOf('@');
-String realHdfsPath = uri.substring(0, cut) + "/" + 
KYLIN_CONF_PROPERTIES_FILE;
-try {
-config = new KylinConfig();
-fs = HadoopUtil.getFileSystem(realHdfsPath);
-InputStream is = fs.open(new Path(realHdfsPath));
-Properties prop = streamToProps(is);
-config.reloadKylinConfig(prop);
-} catch (IOException e) {
-throw new RuntimeException(e);
-}
-return config;
-}
-
 if (uriType == UriType.LOCAL_FOLDER) {
 KylinConfig config = new KylinConfig();
 config.setMetadataUrl(uri);
@@ -241,7 +219,7 @@ public class KylinConfig extends KylinConfigBase {

[4/4] kylin git commit: KYLIN-2814 close response for RestClient's request

[billyliu]

KYLIN-2814 close response for RestClient's request


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f80488e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f80488e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f80488e

Branch: refs/heads/KYLIN-2814
Commit: 3f80488e35d6c9b971833d64257040b938b689fd
Parents: 5c1aab1
Author: Billy Liu 
Authored: Sun Aug 27 12:47:33 2017 +0800
Committer: Billy Liu 
Committed: Thu Aug 31 15:38:31 2017 +0800

--
 .../kylin/common/restclient/RestClient.java | 112 +++--
 .../kylin/restclient/ITRestClientTest.java  | 120 ++-
 2 files changed, 135 insertions(+), 97 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3f80488e/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java 
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index efd0438..6a89b9d 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -37,7 +37,7 @@ import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.entity.StringEntity;
+import org.apache.http.client.utils.HttpClientUtils;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.params.BasicHttpParams;
@@ -129,12 +129,8 @@ public class RestClient {
 String msg = EntityUtils.toString(response.getEntity());
 throw new IOException("Invalid response " + 
response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" 
+ msg);
 }
-} catch (Exception ex) {
-throw new IOException(ex);
 } finally {
-if(response != null){
-EntityUtils.consume(response.getEntity());
-}
+HttpClientUtils.closeQuietly(response);
 request.releaseConnection();
 }
 }
@@ -142,8 +138,10 @@ public class RestClient {
 public String getKylinProperties() throws IOException {
 String url = baseUrl + "/admin/config";
 HttpGet request = new HttpGet(url);
+
+HttpResponse response = null;
 try {
-HttpResponse response = client.execute(request);
+response = client.execute(request);
 String msg = EntityUtils.toString(response.getEntity());
 Map map = JsonUtil.readValueAsMap(msg);
 msg = map.get("config");
@@ -152,83 +150,27 @@ public class RestClient {
 throw new IOException("Invalid response " + 
response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" 
+ msg);
 return msg;
 } finally {
+HttpClientUtils.closeQuietly(response);
 request.releaseConnection();
 }
 }
 
-public boolean enableCache() throws IOException {
-return setCache(true);
-}
-
-public boolean disableCache() throws IOException {
-return setCache(false);
-}
-
-public boolean buildCube(String cubeName, long startTime, long endTime, 
String buildType) throws Exception {
-String url = baseUrl + "/cubes/" + cubeName + "/build";
-HttpPut put = newPut(url);
-HashMap paraMap = new HashMap();
-paraMap.put("startTime", startTime + "");
-paraMap.put("endTime", endTime + "");
-paraMap.put("buildType", buildType);
-String jsonMsg = new ObjectMapper().writeValueAsString(paraMap);
-put.setEntity(new StringEntity(jsonMsg, "UTF-8"));
-HttpResponse response = client.execute(put);
-String result = getContent(response);
-if (response.getStatusLine().getStatusCode() != 200) {
-throw new IOException("Invalid response " + 
response.getStatusLine().getStatusCode() + " with build cube url " + url + "\n" 
+ jsonMsg);
-} else {
-return true;
-}
-}
-
-public boolean disableCube(String cubeName) throws Exception {
-return changeCubeStatus(baseUrl + "/cubes/" + cubeName + "/disable");
-}
-
-public boolean enableCube(String cubeName) throws Exception {
-return changeCubeStatus(baseUrl + "/cubes/" + cubeName + "/enable");
-}
-
-public boolean purgeCube(String cubeName) throws 

[2/4] kylin git commit: KYLIN-2811, fix NPE

[billyliu]

KYLIN-2811, fix NPE

Signed-off-by: Hongbin Ma 


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/727920b4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/727920b4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/727920b4

Branch: refs/heads/KYLIN-2814
Commit: 727920b4d3642aaa3657d90b7f3dce7dcdd68fe2
Parents: a1c234a
Author: Cheng Wang 
Authored: Mon Aug 28 12:47:58 2017 +0800
Committer: Hongbin Ma 
Committed: Mon Aug 28 14:41:26 2017 +0800

--
 .../kylin/engine/spark/SparkCubingByLayer.java  | 43 +++-
 1 file changed, 24 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/727920b4/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
--
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index a3a6ad0..a03e238 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -100,7 +100,6 @@ public class SparkCubingByLayer extends AbstractApplication 
implements Serializa
 .withDescription("Configuration Path").create("confPath");
 
 private Options options;
-private static String metaUrl;
 
 public SparkCubingByLayer() {
 options = new Options();
@@ -117,7 +116,7 @@ public class SparkCubingByLayer extends AbstractApplication 
implements Serializa
 return options;
 }
 
-public static KylinConfig getKylinConfigForExecutor() {
+public static KylinConfig getKylinConfigForExecutor(String metaUrl) {
 File file = new 
File(SparkFiles.get(KylinConfig.KYLIN_CONF_PROPERTIES_FILE));
 String confPath = file.getParentFile().getAbsolutePath();
 System.setProperty(KylinConfig.KYLIN_CONF, confPath);
@@ -128,7 +127,7 @@ public class SparkCubingByLayer extends AbstractApplication 
implements Serializa
 
 @Override
 protected void execute(OptionsHelper optionsHelper) throws Exception {
-metaUrl = optionsHelper.getOptionValue(OPTION_META_URL);
+String metaUrl = optionsHelper.getOptionValue(OPTION_META_URL);
 String hiveTable = optionsHelper.getOptionValue(OPTION_INPUT_TABLE);
 String cubeName = optionsHelper.getOptionValue(OPTION_CUBE_NAME);
 String segmentId = optionsHelper.getOptionValue(OPTION_SEGMENT_ID);
@@ -178,17 +177,17 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 
 // encode with dimension encoding, transform to  
RDD
 final JavaPairRDD encodedBaseRDD = 
intermediateTable.javaRDD()
-.mapToPair(new EncodeBaseCuboid(cubeName, segmentId));
+.mapToPair(new EncodeBaseCuboid(cubeName, segmentId, metaUrl));
 
 Long totalCount = 0L;
 if (envConfig.isSparkSanityCheckEnabled()) {
 totalCount = encodedBaseRDD.count();
 }
 
-final BaseCuboidReducerFunction2 baseCuboidReducerFunction = new 
BaseCuboidReducerFunction2(cubeName);
+final BaseCuboidReducerFunction2 baseCuboidReducerFunction = new 
BaseCuboidReducerFunction2(cubeName, metaUrl);
 BaseCuboidReducerFunction2 reducerFunction2 = 
baseCuboidReducerFunction;
 if (allNormalMeasure == false) {
-reducerFunction2 = new CuboidReducerFunction2(cubeName, needAggr);
+reducerFunction2 = new CuboidReducerFunction2(cubeName, metaUrl, 
needAggr);
 }
 
 final int totalLevels = cubeDesc.getBuildLevel();
@@ -202,18 +201,18 @@ public class SparkCubingByLayer extends 
AbstractApplication implements Serializa
 Configuration confOverwrite = new 
Configuration(sc.hadoopConfiguration());
 confOverwrite.set("dfs.replication", "2"); // cuboid intermediate 
files, replication=2
 
-saveToHDFS(allRDDs[0], cubeName, cubeSegment, outputPath, 0, 
confOverwrite);
+saveToHDFS(allRDDs[0], cubeName, metaUrl, cubeSegment, outputPath, 0, 
confOverwrite);
 
 // aggregate to ND cuboids
 for (level = 1; level <= totalLevels; level++) {
 long levelRddSize = SizeEstimator.estimate(allRDDs[level - 1]) / 
(1024 * 1024);
 partition = estimateRDDPartitionNum(level, cubeStatsReader, 
envConfig, (int) levelRddSize);
-allRDDs[level] = allRDDs[level - 1].flatMapToPair(new 
CuboidFlatMap(cubeName, segmentId))
+allRDDs[level] = allRDDs[level - 1].flatMapToPair(new 
CuboidFlatMap(cubeName, segmentId, 

kylin git commit: APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job priority

[nju_yaho]

Repository: kylin
Updated Branches:
  refs/heads/yaho-cube-planner 0b12df776 -> d216eba3e


APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job 
priority


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d216eba3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d216eba3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d216eba3

Branch: refs/heads/yaho-cube-planner
Commit: d216eba3e10af99cb58b9ca45582d2bf5c8a0a37
Parents: 0b12df7
Author: Zhong 
Authored: Thu Aug 31 14:56:19 2017 +0800
Committer: Zhong 
Committed: Thu Aug 31 14:56:19 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   8 ++
 .../kylin/job/engine/JobEngineConfig.java   |  14 +++
 .../kylin/job/execution/AbstractExecutable.java |   9 ++
 .../job/execution/CheckpointExecutable.java |   7 ++
 .../job/execution/DefaultChainedExecutable.java |   7 ++
 .../job/impl/threadpool/DefaultScheduler.java   | 111 ++-
 .../org/apache/kylin/engine/mr/CubingJob.java   |  19 +++-
 7 files changed, 171 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d216eba3/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d66d7ce..53bb2ad 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -503,6 +503,14 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.job.scheduler.default", 
"0"));
 }
 
+public boolean getSchedulerPriorityConsidered() {
+return 
Boolean.parseBoolean(getOptional("kylin.job.scheduler.priority-considered", 
"false"));
+}
+
+public Integer getSchedulerPriorityBarFetchFromQueue() {
+return 
Integer.parseInt(getOptional("kylin.job.scheduler.priority-bar-fetch-from-queue",
 "20"));
+}
+
 public Integer getSchedulerPollIntervalSecond() {
 return 
Integer.parseInt(getOptional("kylin.job.scheduler.poll-interval-second", "30"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d216eba3/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java 
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 6890557..9ba602f 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -105,6 +105,20 @@ public class JobEngineConfig {
 return config;
 }
 
+/**
+ * @return if consider job priority when scheduling jobs
+ * */
+public boolean getJobPriorityConsidered() {
+return config.getSchedulerPriorityConsidered();
+}
+
+/**
+ * @return the priority bar for fetching jobs from job priority queue
+ */
+public int getFetchQueuePriorityBar() {
+return config.getSchedulerPriorityBarFetchFromQueue();
+}
+
 public String getHdfsWorkingDirectory() {
 return config.getHdfsWorkingDirectory();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d216eba3/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 30b6421..a37cdc9 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -44,6 +44,8 @@ import com.google.common.collect.Maps;
  */
 public abstract class AbstractExecutable implements Executable, Idempotent {
 
+public static final Integer DEFAULT_PRIORITY = 10;
+
 protected static final String SUBMITTER = "submitter";
 protected static final String NOTIFY_LIST = "notify_list";
 protected static final String START_TIME = "startTime";
@@ -389,6 +391,13 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 return output.getState() == ExecutableState.READY;
 }
 
+/**
+ * The larger the value, the higher priority
+ * */
+public int getDefaultPriority() {
+return DEFAULT_PRIORITY;
+}
+
 /*
 

[kylin] Git Push Summary [forced push!] [Forced Update!]

[nju_yaho]

Repository: kylin
Updated Branches:
  refs/heads/yaho-cube-planner 214ddfefa -> 0b12df776 (forced update)

kylin git commit: APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job priority

[nju_yaho]

Repository: kylin
Updated Branches:
  refs/heads/yaho-cube-planner 0b12df776 -> 214ddfefa


APACHE-KYLIN-2735: Introduce an option to make job scheduler consider job 
priority


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/214ddfef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/214ddfef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/214ddfef

Branch: refs/heads/yaho-cube-planner
Commit: 214ddfefaeddc1b08a4963f300cf2e2277603d0b
Parents: 0b12df7
Author: Zhong 
Authored: Thu Aug 31 14:01:26 2017 +0800
Committer: Zhong 
Committed: Thu Aug 31 14:01:26 2017 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|   4 +
 .../kylin/job/engine/JobEngineConfig.java   |   7 ++
 .../kylin/job/execution/AbstractExecutable.java |   9 ++
 .../job/execution/CheckpointExecutable.java |   7 ++
 .../job/execution/DefaultChainedExecutable.java |   7 ++
 .../job/impl/threadpool/DefaultScheduler.java   | 112 ++-
 .../org/apache/kylin/engine/mr/CubingJob.java   |  19 +++-
 7 files changed, 161 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/214ddfef/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d66d7ce..1419a48 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -503,6 +503,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Integer.parseInt(getOptional("kylin.job.scheduler.default", 
"0"));
 }
 
+public boolean getSchedulerPriorityConsidered() {
+return 
Boolean.parseBoolean(getOptional("kylin.job.scheduler.priority-considered", 
"false"));
+}
+
 public Integer getSchedulerPollIntervalSecond() {
 return 
Integer.parseInt(getOptional("kylin.job.scheduler.poll-interval-second", "30"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/214ddfef/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java 
b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 6890557..baa2aaa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -105,6 +105,13 @@ public class JobEngineConfig {
 return config;
 }
 
+/**
+ * @return if consider job priority when scheduling jobs
+ * */
+public boolean getJobPriorityConsidered() {
+return config.getSchedulerPriorityConsidered();
+}
+
 public String getHdfsWorkingDirectory() {
 return config.getHdfsWorkingDirectory();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/214ddfef/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 30b6421..a37cdc9 100644
--- 
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ 
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -44,6 +44,8 @@ import com.google.common.collect.Maps;
  */
 public abstract class AbstractExecutable implements Executable, Idempotent {
 
+public static final Integer DEFAULT_PRIORITY = 10;
+
 protected static final String SUBMITTER = "submitter";
 protected static final String NOTIFY_LIST = "notify_list";
 protected static final String START_TIME = "startTime";
@@ -389,6 +391,13 @@ public abstract class AbstractExecutable implements 
Executable, Idempotent {
 return output.getState() == ExecutableState.READY;
 }
 
+/**
+ * The larger the value, the higher priority
+ * */
+public int getDefaultPriority() {
+return DEFAULT_PRIORITY;
+}
+
 /*
 * discarded is triggered by JobService, the Scheduler is not awake of that
 *

http://git-wip-us.apache.org/repos/asf/kylin/blob/214ddfef/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java
--
diff --git 
a/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java