[2/3] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce9fc5f5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 7674bf4..f53f4a9 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -32,15 +32,14 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -72,7 +71,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableName;
final StorageURL metadataUrl;
-Connection getConnection() throws IOException {
+HConnection getConnection() throws IOException {
return HBaseConnection.get(metadataUrl);
}
@@ -134,7 +133,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllInOneTableName());
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -252,12 +251,13 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
+table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -265,7 +265,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content,
long oldTS, long newTS) throws IOException, IllegalStateException {
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -278,6 +278,8 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " +
resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
+table.flushCommits();
+
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -286,7 +288,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllInOneTableName());
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -299,6 +301,7 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
+table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -319,7 +322,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean
fetchTimestamp) throws IOException {
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllI
[3/3] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API
KYLIN-2307 Create a branch for master with HBase 0.98 API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce9fc5f5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce9fc5f5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce9fc5f5
Branch: refs/heads/2.0.x-hbase0.98
Commit: ce9fc5f5a449f1e721f660b86c1097ad0ad886a2
Parents: ad67fd7
Author: lidongsjtu
Authored: Mon Jan 23 13:17:37 2017 +0800
Committer: Hongbin Ma
Committed: Sat May 20 14:07:18 2017 +0800
--
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 25
dev-support/test_all_against_hdp_2_4_0_0_169.sh | 25
.../sandbox/capacity-scheduler.xml | 17 ++-
examples/test_case_data/sandbox/core-site.xml | 28 +---
examples/test_case_data/sandbox/hbase-site.xml | 119 +
examples/test_case_data/sandbox/hdfs-site.xml | 84 +---
examples/test_case_data/sandbox/hive-site.xml | 89 +
examples/test_case_data/sandbox/mapred-site.xml | 57 +++--
examples/test_case_data/sandbox/yarn-site.xml | 127 +++
.../kylin/provision/BuildCubeWithEngine.java| 17 +--
pom.xml | 117 +
.../kylin/rest/security/AclHBaseStorage.java| 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 +++---
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 ++--
.../apache/kylin/rest/service/CubeService.java | 35 +++--
.../apache/kylin/rest/service/QueryService.java | 24 ++--
.../apache/kylin/rest/service/UserService.java | 17 +--
.../kylin/storage/hbase/HBaseConnection.java| 46 +++
.../kylin/storage/hbase/HBaseResourceStore.java | 33 ++---
.../storage/hbase/cube/SimpleHBaseStore.java| 20 +--
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 +--
.../storage/hbase/steps/DeprecatedGCStep.java | 24 ++--
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--
.../storage/hbase/util/CleanHtableCLI.java | 12 +-
.../storage/hbase/util/CubeMigrationCLI.java| 37 +++---
.../hbase/util/CubeMigrationCheckCLI.java | 17 +--
.../hbase/util/DeployCoprocessorCLI.java| 27 ++--
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 ++---
.../kylin/storage/hbase/util/HBaseClean.java| 18 ++-
.../hbase/util/HBaseRegionSizeCalculator.java | 35 +++--
.../kylin/storage/hbase/util/HBaseUsage.java| 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 ++---
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 +--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 +--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-
.../storage/hbase/util/StorageCleanupJob.java | 20 ++-
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 +--
tool/pom.xml| 10 --
.../org/apache/kylin/tool/CubeMigrationCLI.java | 16 ++-
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
48 files changed, 597 insertions(+), 872 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce9fc5f5/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100755
index 000..f7780dd
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install -DskipTests 2>&1 | tee mci.log
+mvn verify
[1/3] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/2.0.x-hbase0.98 129fc898e -> ce9fc5f5a (forced update)
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce9fc5f5/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
--
diff --git
a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index f52fc3e..19e5db0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -232,9 +231,9 @@ public class ExtendCubeToHybridCLI {
Serializer projectSerializer = new
JsonSerializer(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath,
ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
-Table aclHtable = null;
+HTableInterface aclHtable = null;
try {
-aclHtable =
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix()
+ "_acl"));
+aclHtable =
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix()
+ "_acl");
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,6 +253,7 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
+aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
kylin git commit: minor, more stable ITZookeeperDistributedLockTest
Repository: kylin
Updated Branches:
refs/heads/master 0bdd1c219 -> fd3bfedb5
minor, more stable ITZookeeperDistributedLockTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fd3bfedb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fd3bfedb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fd3bfedb
Branch: refs/heads/master
Commit: fd3bfedb50f488d6679b9a01bc17e214622b8838
Parents: 0bdd1c2
Author: Yang Li
Authored: Sat May 20 14:00:06 2017 +0800
Committer: Yang Li
Committed: Sat May 20 14:00:06 2017 +0800
--
.../kylin/storage/hbase/ITZookeeperDistributedLockTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd3bfedb/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
index 8460cb9..48d6736 100644
---
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
+++
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
@@ -113,12 +113,12 @@ public class ITZookeeperDistributedLockTest extends
HBaseMetadataTestCase {
new Thread() {
@Override
public void run() {
-d.lock(path, 1);
+d.lock(path, 15000);
}
}.start();
c.unlock(path);
-Thread.sleep(1);
+Thread.sleep(2);
assertTrue(c.isLocked(path));
assertEquals(d.getClient(), d.peekLock(path));
[7/7] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API
KYLIN-2307 Create a branch for master with HBase 0.98 API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/129fc898
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/129fc898
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/129fc898
Branch: refs/heads/2.0.x-hbase0.98
Commit: 129fc898e626363760492e9af17d05f4128c129b
Parents: ad67fd7
Author: lidongsjtu
Authored: Mon Jan 23 13:17:37 2017 +0800
Committer: Hongbin Ma
Committed: Sat May 20 13:32:06 2017 +0800
--
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 25
dev-support/test_all_against_hdp_2_4_0_0_169.sh | 25
.../sandbox/capacity-scheduler.xml | 17 ++-
examples/test_case_data/sandbox/core-site.xml | 28 +---
examples/test_case_data/sandbox/hbase-site.xml | 119 +
examples/test_case_data/sandbox/hdfs-site.xml | 84 +---
examples/test_case_data/sandbox/hive-site.xml | 89 +
examples/test_case_data/sandbox/mapred-site.xml | 57 +++--
examples/test_case_data/sandbox/yarn-site.xml | 127 +++
.../kylin/provision/BuildCubeWithEngine.java| 17 +--
pom.xml | 117 +
.../kylin/rest/security/AclHBaseStorage.java| 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 +++---
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 ++--
.../apache/kylin/rest/service/CubeService.java | 35 +++--
.../apache/kylin/rest/service/QueryService.java | 24 ++--
.../apache/kylin/rest/service/UserService.java | 17 +--
.../kylin/storage/hbase/HBaseConnection.java| 46 +++
.../kylin/storage/hbase/HBaseResourceStore.java | 33 ++---
.../storage/hbase/cube/SimpleHBaseStore.java| 20 +--
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 +--
.../storage/hbase/steps/DeprecatedGCStep.java | 24 ++--
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--
.../storage/hbase/util/CleanHtableCLI.java | 12 +-
.../storage/hbase/util/CubeMigrationCLI.java| 37 +++---
.../hbase/util/CubeMigrationCheckCLI.java | 17 +--
.../hbase/util/DeployCoprocessorCLI.java| 27 ++--
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 ++---
.../kylin/storage/hbase/util/HBaseClean.java| 18 ++-
.../hbase/util/HBaseRegionSizeCalculator.java | 35 +++--
.../kylin/storage/hbase/util/HBaseUsage.java| 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 ++---
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 +--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 +--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-
.../storage/hbase/util/StorageCleanupJob.java | 20 ++-
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 +--
tool/pom.xml| 10 --
.../org/apache/kylin/tool/CubeMigrationCLI.java | 16 ++-
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
48 files changed, 597 insertions(+), 872 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/129fc898/dev-support/test_all_against_hdp_2_2_4_2_2.sh
--
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100755
index 000..f7780dd
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install -DskipTests 2>&1 | tee mci.log
+mvn verify
[2/7] kylin git commit: KYLIN-2630 NPE when a subquery joins another lookup tables
KYLIN-2630 NPE when a subquery joins another lookup tables
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/19f9c685
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/19f9c685
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/19f9c685
Branch: refs/heads/2.0.x-hbase0.98
Commit: 19f9c6852d20f822a204963cbdda24518d9ff1da
Parents: 79db276
Author: Hongbin Ma
Authored: Thu May 18 20:54:45 2017 +0800
Committer: Hongbin Ma
Committed: Thu May 18 20:55:09 2017 +0800
--
.../resources/query/sql_subquery/query30.sql| 15 ++
.../resources/query/sql_subquery/query31.sql| 19
.../apache/kylin/query/relnode/OLAPJoinRel.java | 12
.../org/apache/kylin/query/relnode/OLAPRel.java | 31 +++-
.../kylin/query/relnode/OLAPTableScan.java | 18 ++--
5 files changed, 72 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/19f9c685/kylin-it/src/test/resources/query/sql_subquery/query30.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query30.sql
b/kylin-it/src/test/resources/query/sql_subquery/query30.sql
new file mode 100644
index 000..3b0dfbf
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query30.sql
@@ -0,0 +1,15 @@
+
+SELECT t1.cal_dt, t1.sum_price,t1.lstg_site_id
+FROM (
+ select cal_dt, lstg_site_id, sum(price) as sum_price
+ from test_kylin_fact
+ group by cal_dt, lstg_site_id
+
+) t1
+
+inner JOIN edw.test_cal_dt as test_cal_dt
+on t1.cal_dt=test_cal_dt.cal_dt
+
+inner JOIN edw.test_sites as test_sites
+on t1.lstg_site_id = test_sites.site_id
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/19f9c685/kylin-it/src/test/resources/query/sql_subquery/query31.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query31.sql
b/kylin-it/src/test/resources/query/sql_subquery/query31.sql
new file mode 100644
index 000..fafc01f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query31.sql
@@ -0,0 +1,19 @@
+
+SELECT t1.week_beg_dt, t1.sum_price,t1.lstg_site_id
+FROM (
+ select test_cal_dt.week_beg_dt, sum(price) as sum_price, lstg_site_id
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ group by test_cal_dt.week_beg_dt, lstg_site_id
+) t1
+inner JOIN edw.test_cal_dt as test_cal_dt
+on t1.week_beg_dt=test_cal_dt.week_beg_dt
+ inner JOIN edw.test_sites as test_sites
+
+ on t1.lstg_site_id = test_sites.site_id
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/19f9c685/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 60b5712..1b5970c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -140,12 +140,24 @@ public class OLAPJoinRel extends EnumerableJoin
implements OLAPRel {
implementor.freeContext();
}
}
+
+if (leftHasSubquery) {
+// After KYLIN-2579, leftHasSubquery means right side have to be
separate olap context
+implementor.setNewOLAPContextRequired(true);
+}
+
implementor.fixSharedOlapTableScanOnTheRight(this);
implementor.visitChild(this.right, this);
if (this.context != implementor.getContext() || ((OLAPRel)
this.right).hasSubQuery()) {
this.hasSubQuery = true;
rightHasSubquery = true;
// if child is also an OLAPJoin, then the context has already been
popped
+
+if (leftHasSubquery) {
+
Preconditions.checkState(!implementor.isNewOLAPContextRequired());//should have
been satisfied
+Preconditions.checkState(this.context !=
implementor.getContext(), "missing a new olapcontext");
+}
+
if (this.context != implementor.getContext()) {
implementor.freeContext();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19f9c685/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
--
diff
[4/7] kylin git commit: KYLIN-2631 Seek to next model when no cube in current model satisfies query
KYLIN-2631 Seek to next model when no cube in current model satisfies query
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad67fd76
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad67fd76
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad67fd76
Branch: refs/heads/2.0.x-hbase0.98
Commit: ad67fd763237306ddba0d0946ab024fcdb7ab604
Parents: 0f68486
Author: Hongbin Ma
Authored: Fri May 19 21:12:26 2017 +0800
Committer: Hongbin Ma
Committed: Fri May 19 21:12:34 2017 +0800
--
.../apache/kylin/metadata/model/TblColRef.java | 43 ++--
.../kylin/metadata/realization/SQLDigest.java | 2 +
.../gtrecord/GTCubeStorageQueryBase.java| 3 -
.../kylin/query/relnode/OLAPTableScan.java | 13 +-
.../relnode/OLAPToEnumerableConverter.java | 11 +-
.../kylin/query/routing/ModelChooser.java | 219 --
.../apache/kylin/query/routing/QueryRouter.java | 2 +-
.../kylin/query/routing/RealizationChooser.java | 224 +++
8 files changed, 271 insertions(+), 246 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad67fd76/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index c74f4e8..1410841 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -56,7 +56,7 @@ public class TblColRef implements Serializable {
public static TblColRef newInnerColumn(String columnName,
InnerDataTypeEnum dataType) {
return newInnerColumn(columnName, dataType, null);
}
-
+
// used by projection rewrite, see OLAPProjectRel
public static TblColRef newInnerColumn(String columnName,
InnerDataTypeEnum dataType, String parserDescription) {
ColumnDesc column = new ColumnDesc();
@@ -68,27 +68,30 @@ public class TblColRef implements Serializable {
colRef.parserDescription = parserDescription;
return colRef;
}
-
+
private static final DataModelDesc UNKNOWN_MODEL = new DataModelDesc();
static {
UNKNOWN_MODEL.setName("UNKNOWN_MODEL");
}
-
+
public static TableRef tableForUnknownModel(String tempTableAlias,
TableDesc table) {
return new TableRef(UNKNOWN_MODEL, tempTableAlias, table);
}
-
+
public static TblColRef columnForUnknownModel(TableRef table, ColumnDesc
colDesc) {
checkArgument(table.getModel() == UNKNOWN_MODEL);
return new TblColRef(table, colDesc);
}
-
+
public static void fixUnknownModel(DataModelDesc model, String alias,
TblColRef col) {
checkArgument(col.table.getModel() == UNKNOWN_MODEL ||
col.table.getModel() == model);
TableRef tableRef = model.findTable(alias);
checkArgument(tableRef.getTableDesc() == col.column.getTable());
-col.table = tableRef;
-col.identity = null;
+col.fixTableRef(tableRef);
+}
+
+public static void unfixUnknownModel(TblColRef col) {
+col.unfixTableRef();
}
// for test mainly
@@ -101,10 +104,11 @@ public class TblColRef implements Serializable {
desc.init(table);
return new TblColRef(desc);
}
-
+
//
private TableRef table;
+private TableRef backupTable;// only used in fixTableRef()
private ColumnDesc column;
private String identity;
private String parserDescription;
@@ -112,13 +116,24 @@ public class TblColRef implements Serializable {
TblColRef(ColumnDesc column) {
this.column = column;
}
-
+
TblColRef(TableRef table, ColumnDesc column) {
checkArgument(table.getTableDesc() == column.getTable());
this.table = table;
this.column = column;
}
-
+
+public void fixTableRef(TableRef tableRef) {
+this.backupTable = this.table;
+this.table = tableRef;
+this.identity = null;
+}
+
+public void unfixTableRef() {
+this.table = backupTable;
+this.identity = null;
+}
+
public ColumnDesc getColumnDesc() {
return column;
}
@@ -130,15 +145,15 @@ public class TblColRef implements Serializable {
public TableRef getTableRef() {
return table;
}
-
+
public boolean isQualified() {
return table != null;
}
-
+
public String getTableAlias() {
return table != null ? table.getAlias() : "UNKNOWN_ALIAS";
}
-
[6/7] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API
http://git-wip-us.apache.org/repos/asf/kylin/blob/129fc898/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 7674bf4..f53f4a9 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -32,15 +32,14 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -72,7 +71,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableName;
final StorageURL metadataUrl;
-Connection getConnection() throws IOException {
+HConnection getConnection() throws IOException {
return HBaseConnection.get(metadataUrl);
}
@@ -134,7 +133,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllInOneTableName());
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -252,12 +251,13 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
+table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -265,7 +265,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content,
long oldTS, long newTS) throws IOException, IllegalStateException {
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -278,6 +278,8 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " +
resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
+table.flushCommits();
+
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -286,7 +288,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllInOneTableName());
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -299,6 +301,7 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
+table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -319,7 +322,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean
fetchTimestamp) throws IOException {
-Table table =
getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+HTableInterface table =
getConnection().getTable(getAllI
[1/7] kylin git commit: KYLIN-2624 introduce StorageURL, add timeout to HBaseResourceStore [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/2.0.x-hbase0.98 5d4a5283c -> 129fc898e (forced update)
KYLIN-2624 introduce StorageURL, add timeout to HBaseResourceStore
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/79db2768
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/79db2768
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/79db2768
Branch: refs/heads/2.0.x-hbase0.98
Commit: 79db2768bc517d9a8e64f657baff10e1914caa7c
Parents: d03802c
Author: Li Yang
Authored: Wed May 17 22:27:52 2017 +
Committer: Li Yang
Committed: Wed May 17 22:28:20 2017 +
--
.../apache/kylin/common/KylinConfigBase.java| 30 ++-
.../org/apache/kylin/common/StorageURL.java | 183 +++
.../common/persistence/FileResourceStore.java | 2 +-
.../apache/kylin/common/KylinConfigTest.java| 9 +-
.../org/apache/kylin/common/StorageURLTest.java | 103 +++
.../persistence/LocalFileResourceStoreTest.java | 3 +-
.../common/persistence/ResourceStoreTest.java | 2 +-
.../apache/kylin/dict/DictionaryManager.java| 3 +-
.../apache/kylin/storage/StorageContext.java| 7 +-
.../kylin/engine/mr/steps/CubeReducerTest.java | 2 +-
.../engine/mr/steps/NDCuboidMapperTest.java | 2 +-
examples/test_case_data/localmeta/UUID | Bin 0 -> 38 bytes
.../kylin/job/BaseTestDistributedScheduler.java | 8 +-
.../kylin/storage/hbase/ITStorageTest.java | 3 +-
.../apache/kylin/query/schema/OLAPSchema.java | 5 +-
.../rest/security/MockAclHBaseStorage.java | 5 +-
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/QueryService.java | 11 +-
.../kylin/storage/hbase/HBaseConnection.java| 26 ++-
.../kylin/storage/hbase/HBaseResourceStore.java | 36 ++--
.../storage/hbase/util/CubeMigrationCLI.java| 20 +-
.../hbase/util/GridTableHBaseBenchmark.java | 3 +-
.../hbase/util/ZookeeperDistributedJobLock.java | 14 +-
.../kylin/storage/hdfs/HDFSResourceStore.java | 33 +---
.../hbase/steps/CubeHFileMapper2Test.java | 4 +-
.../storage/hbase/steps/MockupMapContext.java | 2 +-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 20 +-
27 files changed, 410 insertions(+), 135 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/79db2768/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 09bbb72..9c698ac 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -167,7 +167,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String toString() {
-return getMetadataUrl();
+return getMetadataUrl().toString();
}
//
@@ -203,8 +203,8 @@ abstract public class KylinConfigBase implements
Serializable {
// METADATA
//
-public String getMetadataUrl() {
-return getOptional("kylin.metadata.url");
+public StorageURL getMetadataUrl() {
+return StorageURL.valueOf(getOptional("kylin.metadata.url", ""));
}
// for test only
@@ -213,14 +213,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String getMetadataUrlPrefix() {
-String metadataUrl = getMetadataUrl();
-String defaultPrefix = "kylin_metadata";
-
-int cut = metadataUrl.lastIndexOf('@');
-if (cut > 0) {
-return metadataUrl.substring(0, cut);
-}
-return defaultPrefix;
+return getMetadataUrl().getIdentifier();
}
public String[] getRealizationProviders() {
@@ -543,8 +536,14 @@ abstract public class KylinConfigBase implements
Serializable {
return Integer.parseInt(getOptional("kylin.storage.default", "2"));
}
-public String getStorageUrl() {
-return getOptional("kylin.storage.url");
+public StorageURL getStorageUrl() {
+String url = getOptional("kylin.storage.url", "default@hbase");
+
+// for backward compatibility
+if ("hbase".equals(url))
+url = "default@hbase";
+
+return StorageURL.valueOf(url);
}
public String getHBaseClusterFs() {
@@ -972,10 +971,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String getResourceStoreImpl() {
-String metadataUrl
[3/7] kylin git commit: minor enhance to KYLIN-2624
minor enhance to KYLIN-2624
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f68486f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f68486f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f68486f
Branch: refs/heads/2.0.x-hbase0.98
Commit: 0f68486f2e8aba0d8b3abd9b19987070d7250144
Parents: 19f9c68
Author: Li Yang
Authored: Fri May 19 14:31:23 2017 +0800
Committer: Li Yang
Committed: Fri May 19 14:31:29 2017 +0800
--
.../apache/kylin/common/util/LocalFileMetadataTestCase.java | 7 +++
1 file changed, 7 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f68486f/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index 4a24941..8f47fac 100644
---
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -77,6 +77,13 @@ public class LocalFileMetadataTestCase extends
AbstractKylinTestCase {
cleanAfterClass();
}
+protected String getLocalWorkingDirectory() {
+String dir =
KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
+if (dir.startsWith("file://"))
+dir = dir.substring("file://".length());
+return new File(dir).getAbsolutePath();
+}
+
protected ResourceStore getStore() {
return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
}
[5/7] kylin git commit: KYLIN-2307 Create a branch for master with HBase 0.98 API
http://git-wip-us.apache.org/repos/asf/kylin/blob/129fc898/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
--
diff --git
a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index f52fc3e..19e5db0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -232,9 +231,9 @@ public class ExtendCubeToHybridCLI {
Serializer projectSerializer = new
JsonSerializer(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath,
ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
-Table aclHtable = null;
+HTableInterface aclHtable = null;
try {
-aclHtable =
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix()
+ "_acl"));
+aclHtable =
HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix()
+ "_acl");
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,6 +253,7 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
+aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
kylin git commit: KYLIN-2631 Seek to next model when no cube in current model satisfies query
Repository: kylin
Updated Branches:
refs/heads/2.0.x 0f68486f2 -> ad67fd763
KYLIN-2631 Seek to next model when no cube in current model satisfies query
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad67fd76
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad67fd76
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad67fd76
Branch: refs/heads/2.0.x
Commit: ad67fd763237306ddba0d0946ab024fcdb7ab604
Parents: 0f68486
Author: Hongbin Ma
Authored: Fri May 19 21:12:26 2017 +0800
Committer: Hongbin Ma
Committed: Fri May 19 21:12:34 2017 +0800
--
.../apache/kylin/metadata/model/TblColRef.java | 43 ++--
.../kylin/metadata/realization/SQLDigest.java | 2 +
.../gtrecord/GTCubeStorageQueryBase.java| 3 -
.../kylin/query/relnode/OLAPTableScan.java | 13 +-
.../relnode/OLAPToEnumerableConverter.java | 11 +-
.../kylin/query/routing/ModelChooser.java | 219 --
.../apache/kylin/query/routing/QueryRouter.java | 2 +-
.../kylin/query/routing/RealizationChooser.java | 224 +++
8 files changed, 271 insertions(+), 246 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad67fd76/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index c74f4e8..1410841 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -56,7 +56,7 @@ public class TblColRef implements Serializable {
public static TblColRef newInnerColumn(String columnName,
InnerDataTypeEnum dataType) {
return newInnerColumn(columnName, dataType, null);
}
-
+
// used by projection rewrite, see OLAPProjectRel
public static TblColRef newInnerColumn(String columnName,
InnerDataTypeEnum dataType, String parserDescription) {
ColumnDesc column = new ColumnDesc();
@@ -68,27 +68,30 @@ public class TblColRef implements Serializable {
colRef.parserDescription = parserDescription;
return colRef;
}
-
+
private static final DataModelDesc UNKNOWN_MODEL = new DataModelDesc();
static {
UNKNOWN_MODEL.setName("UNKNOWN_MODEL");
}
-
+
public static TableRef tableForUnknownModel(String tempTableAlias,
TableDesc table) {
return new TableRef(UNKNOWN_MODEL, tempTableAlias, table);
}
-
+
public static TblColRef columnForUnknownModel(TableRef table, ColumnDesc
colDesc) {
checkArgument(table.getModel() == UNKNOWN_MODEL);
return new TblColRef(table, colDesc);
}
-
+
public static void fixUnknownModel(DataModelDesc model, String alias,
TblColRef col) {
checkArgument(col.table.getModel() == UNKNOWN_MODEL ||
col.table.getModel() == model);
TableRef tableRef = model.findTable(alias);
checkArgument(tableRef.getTableDesc() == col.column.getTable());
-col.table = tableRef;
-col.identity = null;
+col.fixTableRef(tableRef);
+}
+
+public static void unfixUnknownModel(TblColRef col) {
+col.unfixTableRef();
}
// for test mainly
@@ -101,10 +104,11 @@ public class TblColRef implements Serializable {
desc.init(table);
return new TblColRef(desc);
}
-
+
//
private TableRef table;
+private TableRef backupTable;// only used in fixTableRef()
private ColumnDesc column;
private String identity;
private String parserDescription;
@@ -112,13 +116,24 @@ public class TblColRef implements Serializable {
TblColRef(ColumnDesc column) {
this.column = column;
}
-
+
TblColRef(TableRef table, ColumnDesc column) {
checkArgument(table.getTableDesc() == column.getTable());
this.table = table;
this.column = column;
}
-
+
+public void fixTableRef(TableRef tableRef) {
+this.backupTable = this.table;
+this.table = tableRef;
+this.identity = null;
+}
+
+public void unfixTableRef() {
+this.table = backupTable;
+this.identity = null;
+}
+
public ColumnDesc getColumnDesc() {
return column;
}
@@ -130,15 +145,15 @@ public class TblColRef implements Serializable {
public TableRef getTableRef() {
return table;
}
-
+
public boolean isQualified() {
return table != null;
}
-
+
public String getTableAlias() {
r
kylin git commit: KYLIN-2624 pass IT
Repository: kylin
Updated Branches:
refs/heads/master be32f7588 -> 0bdd1c219
KYLIN-2624 pass IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0bdd1c21
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0bdd1c21
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0bdd1c21
Branch: refs/heads/master
Commit: 0bdd1c21981cef4d9e6edafd63b9c5a0fb2a6725
Parents: be32f75
Author: Yang Li
Authored: Sat May 20 07:56:24 2017 +0800
Committer: Yang Li
Committed: Sat May 20 09:38:51 2017 +0800
--
examples/test_case_data/localmeta/UUID | 1 +
.../kylin/job/BaseTestDistributedScheduler.java | 2 +
.../hbase/ITZookeeperDistributedLockTest.java | 257 +++
.../hbase/util/ZookeeperDistributedLock.java| 11 +-
.../util/ITZookeeperDistributedLockTest.java| 254 --
5 files changed, 268 insertions(+), 257 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bdd1c21/examples/test_case_data/localmeta/UUID
--
diff --git a/examples/test_case_data/localmeta/UUID
b/examples/test_case_data/localmeta/UUID
new file mode 100644
index 000..e3f5e6b
--- /dev/null
+++ b/examples/test_case_data/localmeta/UUID
@@ -0,0 +1 @@
+$3591078f-9401-481a-a233-e7755c1142d0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bdd1c21/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index 6343dfa..1ea6507 100644
---
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -113,6 +113,8 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
@AfterClass
public static void after() throws Exception {
+jobLock1.purgeLocks("");
+
if (scheduler1 != null) {
scheduler1.shutdown();
scheduler1 = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/0bdd1c21/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
new file mode 100644
index 000..8460cb9
--- /dev/null
+++
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.kylin.common.lock.DistributedLock;
+import org.apache.kylin.common.lock.DistributedLock.Watcher;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ITZookeeperDistributedLockTest extends HBaseMetadataTestCase {
+private static final Logger logger =
LoggerFactory.getLogger(ITZookeeperDistributedLockTest.class);
+private static final String ZK_PFX = "/test/ZookeeperDistributedLockTest/"
+ new Random().nextInt(1000);
+
+static ZookeeperDistributedLock.Factory factory;
+
+@BeforeClass
+public static void setup() thr
kylin git commit: final final fix
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2624 0206d87e0 -> 667804e22
final final fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/667804e2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/667804e2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/667804e2
Branch: refs/heads/KYLIN-2624
Commit: 667804e224ae250d9b6e4040a2cc476aa34b14f4
Parents: 0206d87
Author: Yang Li
Authored: Sat May 20 09:35:27 2017 +0800
Committer: Yang Li
Committed: Sat May 20 09:35:27 2017 +0800
--
.../apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/667804e2/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
index cec44b7..8460cb9 100644
---
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
+++
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
@@ -176,6 +176,8 @@ public class ITZookeeperDistributedLockTest extends
HBaseMetadataTestCase {
for (int i = 0; i < nClients; i++) {
threads[i].join();
}
+
+Thread.sleep(3000);
// verify counters
assertEquals(0, lockCounter.get() - unlockCounter.get());
kylin git commit: final fix
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2624 12251b6cf -> 0206d87e0
final fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0206d87e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0206d87e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0206d87e
Branch: refs/heads/KYLIN-2624
Commit: 0206d87e0d0cc58e86a0b16721055f65931c97d0
Parents: 12251b6
Author: Yang Li
Authored: Sat May 20 09:00:37 2017 +0800
Committer: Yang Li
Committed: Sat May 20 09:00:37 2017 +0800
--
examples/test_case_data/localmeta/UUID | 1 +
.../apache/kylin/job/BaseTestDistributedScheduler.java | 2 ++
.../storage/hbase/util/ZookeeperDistributedLock.java | 11 ---
3 files changed, 11 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0206d87e/examples/test_case_data/localmeta/UUID
--
diff --git a/examples/test_case_data/localmeta/UUID
b/examples/test_case_data/localmeta/UUID
new file mode 100644
index 000..e3f5e6b
--- /dev/null
+++ b/examples/test_case_data/localmeta/UUID
@@ -0,0 +1 @@
+$3591078f-9401-481a-a233-e7755c1142d0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/0206d87e/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index 6343dfa..1ea6507 100644
---
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -113,6 +113,8 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
@AfterClass
public static void after() throws Exception {
+jobLock1.purgeLocks("");
+
if (scheduler1 != null) {
scheduler1.shutdown();
scheduler1 = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/0206d87e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
index 05d3a5a..db65491 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
@@ -102,7 +102,7 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
public Factory(KylinConfig config) {
this.curator = getZKClient(config);
-this.zkPathBase = dropDoubleSlash("/kylin/" +
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix());
+this.zkPathBase = fixSlash("/kylin/" +
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix());
}
@Override
@@ -284,10 +284,15 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
if (!lockPath.startsWith(zkPathBase))
lockPath = zkPathBase + (lockPath.startsWith("/") ? "" : "/") +
lockPath;
-return dropDoubleSlash(lockPath);
+return fixSlash(lockPath);
}
-public static String dropDoubleSlash(String path) {
+private static String fixSlash(String path) {
+if (!path.startsWith("/"))
+path = "/" + path;
+if (path.endsWith("/"))
+path = path.substring(0, path.length() - 1);
+
for (int n = Integer.MAX_VALUE; n > path.length();) {
n = path.length();
path = path.replace("//", "/");
kylin git commit: move ITZookeeperDistributedLockTest
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2624 3c9607fc4 -> 12251b6cf
move ITZookeeperDistributedLockTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/12251b6c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/12251b6c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/12251b6c
Branch: refs/heads/KYLIN-2624
Commit: 12251b6cfffd1a781faeca3794fa7a16edf2af6a
Parents: 3c9607f
Author: Yang Li
Authored: Sat May 20 08:03:55 2017 +0800
Committer: Yang Li
Committed: Sat May 20 08:03:55 2017 +0800
--
.../hbase/ITZookeeperDistributedLockTest.java | 255 +++
.../util/ITZookeeperDistributedLockTest.java| 254 --
2 files changed, 255 insertions(+), 254 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/12251b6c/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
new file mode 100644
index 000..cec44b7
--- /dev/null
+++
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITZookeeperDistributedLockTest.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.kylin.common.lock.DistributedLock;
+import org.apache.kylin.common.lock.DistributedLock.Watcher;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ITZookeeperDistributedLockTest extends HBaseMetadataTestCase {
+private static final Logger logger =
LoggerFactory.getLogger(ITZookeeperDistributedLockTest.class);
+private static final String ZK_PFX = "/test/ZookeeperDistributedLockTest/"
+ new Random().nextInt(1000);
+
+static ZookeeperDistributedLock.Factory factory;
+
+@BeforeClass
+public static void setup() throws Exception {
+staticCreateTestMetadata();
+factory = new ZookeeperDistributedLock.Factory();
+}
+
+@AfterClass
+public static void after() throws Exception {
+staticCleanupTestMetadata();
+factory.lockForCurrentProcess().purgeLocks(ZK_PFX);
+}
+
+@Test
+public void testBasic() {
+DistributedLock l = factory.lockForCurrentThread();
+String path = ZK_PFX + "/testBasic";
+
+assertTrue(l.isLocked(path) == false);
+assertTrue(l.lock(path));
+assertTrue(l.lock(path));
+assertTrue(l.lock(path));
+assertEquals(l.getClient(), l.peekLock(path));
+assertTrue(l.isLocked(path));
+assertTrue(l.isLockedByMe(path));
+l.unlock(path);
+assertTrue(l.isLocked(path) == false);
+}
+
+@Test
+public void testErrorCases() {
+DistributedLock c = factory.lockForClient("client1");
+DistributedLock d = factory.lockForClient("client2");
+String path = ZK_PFX + "/testErrorCases";
+
+assertTrue(c.isLocked(path) == false);
+assertTrue(d.peekLock(path) == null);
+
+assertTrue(c.lock(path));
+assertTrue(d.lock(path) == false);
+assertTrue(d.isLocked(path) == true);
+assertEquals(c.getClient(), d.peekLock(path));
+
+try {
+d.unlock(path);
+fail();
+} catch (IllegalStateException ex) {
+// expected
+}
+
+c.u
[47/50] [abbrv] kylin git commit: KAP-733 minor UT fix
KAP-733 minor UT fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ce972fd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ce972fd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ce972fd
Branch: refs/heads/KYLIN-2624
Commit: 8ce972fd6475e923af2a5e525d8ddb698d0dbadf
Parents: 2dfc5bb
Author: Li Yang
Authored: Fri May 19 15:50:32 2017 +0800
Committer: liyang-gmt8
Committed: Fri May 19 17:25:55 2017 +0800
--
.../apache/kylin/common/util/LocalFileMetadataTestCase.java| 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce972fd/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index e5e4f8b..a44a55d 100644
---
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -81,7 +81,11 @@ public class LocalFileMetadataTestCase extends
AbstractKylinTestCase {
String dir =
KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
if (dir.startsWith("file://"))
dir = dir.substring("file://".length());
-return dir;
+try {
+return new File(dir).getCanonicalPath();
+} catch (IOException e) {
+throw new RuntimeException(e);
+}
}
protected ResourceStore getStore() {
[10/50] [abbrv] kylin git commit: KYLIN-2583 remove max connection limit
KYLIN-2583 remove max connection limit
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1ba9af70
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1ba9af70
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1ba9af70
Branch: refs/heads/KYLIN-2624
Commit: 1ba9af70a5e8c49dfa3ecbc1e3222cbd46b06cbe
Parents: 8b277d5
Author: Yifan Zhang
Authored: Mon May 8 13:12:41 2017 +0800
Committer: Dong Li
Committed: Mon May 8 15:25:53 2017 +0800
--
.../org/apache/kylin/query/QueryDataSource.java | 5 ++
.../apache/kylin/query/QueryDataSourceTest.java | 50 +++-
2 files changed, 54 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1ba9af70/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
b/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
index 32feea6..9ffdf0e 100644
--- a/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
+++ b/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
@@ -32,6 +32,7 @@ import org.apache.calcite.jdbc.Driver;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbcp.BasicDataSourceFactory;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.log4j.Logger;
@@ -129,11 +130,15 @@ public class QueryDataSource {
}
BasicDataSource ds = null;
+if (StringUtils.isEmpty(props.getProperty("maxActive"))) {
+props.setProperty("maxActive", "-1");
+}
try {
ds = (BasicDataSource)
BasicDataSourceFactory.createDataSource(props);
} catch (Exception e) {
// Basic mode
ds = new BasicDataSource();
+ds.setMaxActive(-1);
}
ds.setUrl("jdbc:calcite:model=" + olapTmp.getAbsolutePath());
ds.setDriverClassName(Driver.class.getName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/1ba9af70/query/src/test/java/org/apache/kylin/query/QueryDataSourceTest.java
--
diff --git
a/query/src/test/java/org/apache/kylin/query/QueryDataSourceTest.java
b/query/src/test/java/org/apache/kylin/query/QueryDataSourceTest.java
index 90d3518..2c06be8 100644
--- a/query/src/test/java/org/apache/kylin/query/QueryDataSourceTest.java
+++ b/query/src/test/java/org/apache/kylin/query/QueryDataSourceTest.java
@@ -18,6 +18,8 @@
package org.apache.kylin.query;
+import java.sql.Connection;
+import java.sql.SQLException;
import java.util.Properties;
import javax.sql.DataSource;
@@ -25,6 +27,7 @@ import javax.sql.DataSource;
import org.apache.calcite.jdbc.Driver;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.DBUtils;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.junit.After;
import org.junit.Assert;
@@ -80,7 +83,52 @@ public class QueryDataSourceTest extends
LocalFileMetadataTestCase {
Assert.assertNotNull(ds3);
Assert.assertEquals(ds1, ds2);
Assert.assertNotEquals(ds1, ds3);
-
+
dsCache.clearCache();
}
+
+@Test(timeout=1)
+public void testMaxConnLimit() throws SQLException {
+KylinConfig config = KylinConfig.getInstanceFromEnv();
+
+// Test with connection limit
+Properties props = new Properties();
+props.setProperty("maxActive", "3");
+props.setProperty("maxWait", "1000");
+DataSource ds1 = QueryDataSource.create("default", config, props);
+Connection ds1Conn1 = ds1.getConnection();
+Connection ds1Conn2 = ds1.getConnection();
+Connection ds1Conn3 = ds1.getConnection();
+try {
+Connection ds1Conn4 = ds1.getConnection();
+DBUtils.closeQuietly(ds1Conn4);
+Assert.fail("This should fail with SQLException");
+} catch (Exception e) {
+Assert.assertTrue(e instanceof SQLException);
+}
+DBUtils.closeQuietly(ds1Conn1);
+DBUtils.closeQuietly(ds1Conn2);
+DBUtils.closeQuietly(ds1Conn3);
+
+// Test with not connection limit
+DataSource ds2 = QueryDataSource.create("default", config);
+Connection ds2Conn1 = ds2.getConnection();
+Connection ds2Conn2 = ds2.getConnection();
+Connection ds2Conn3 = ds2.getConnection();
+Connection ds2Conn4 = ds2.getConnection();
+Connection ds2Conn5 = ds2.getConnec
[01/50] [abbrv] kylin git commit: KYLIN-2586 use random port for CacheServiceTest as fixed port 7777 might have been occupied [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2624 29dd1d24b -> 3c9607fc4 (forced update)
KYLIN-2586 use random port for CacheServiceTest as fixed port might have
been occupied
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/783bb4ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/783bb4ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/783bb4ee
Branch: refs/heads/KYLIN-2624
Commit: 783bb4eecdcd9d455d61009eef27f2508f14d8c8
Parents: d7d468a
Author: Hongbin Ma
Authored: Fri May 5 09:24:42 2017 +0800
Committer: Hongbin Ma
Committed: Fri May 5 09:24:42 2017 +0800
--
.../org/apache/kylin/rest/service/CacheServiceTest.java | 11 ---
1 file changed, 8 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/783bb4ee/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
--
diff --git
a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 7f7b6f0..3601ec1 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
@@ -72,13 +73,17 @@ public class CacheServiceTest extends
LocalFileMetadataTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
staticCreateTestMetadata();
+
+Random random = new Random();
+int port = random.nextInt(1) + 4;
+logger.info("Chosen port for CacheServiceTest is " + port);
configA = KylinConfig.getInstanceFromEnv();
-configA.setProperty("kylin.server.cluster-servers", "localhost:");
+configA.setProperty("kylin.server.cluster-servers", "localhost:" +
port);
configB = KylinConfig.createKylinConfig(configA);
-configB.setProperty("kylin.server.cluster-servers", "localhost:");
+configB.setProperty("kylin.server.cluster-servers", "localhost:" +
port);
configB.setMetadataUrl("../examples/test_metadata");
-server = new Server();
+server = new Server(port);
ServletContextHandler context = new
ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
server.setHandler(context);
[21/50] [abbrv] kylin git commit: KYLIN-2602 Add optional job threshold arg for MetadataCleanupJob
KYLIN-2602 Add optional job threshold arg for MetadataCleanupJob
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3310dcd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3310dcd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3310dcd
Branch: refs/heads/KYLIN-2624
Commit: b3310dcdc0f9ac99006eb4d1a0910de9c43f22fd
Parents: 1264ff3
Author: kangkaisen
Authored: Thu May 11 10:11:40 2017 +0800
Committer: kangkaisen
Committed: Thu May 11 10:13:58 2017 +0800
--
.../org/apache/kylin/tool/MetadataCleanupJob.java | 14 +++---
1 file changed, 11 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b3310dcd/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
--
diff --git a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
index 7040dbb..8e17645 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
@@ -48,14 +48,17 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
@SuppressWarnings("static-access")
private static final Option OPTION_DELETE =
OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete
the unused metadata").create("delete");
+@SuppressWarnings("static-access")
+private static final Option OPTION_THRESHOLD_FOR_JOB =
OptionBuilder.withArgName("jobThreshold").hasArg().isRequired(false).withDescription("Specify
how many days of job metadata keeping. Default 30
days").create("jobThreshold");
+
protected static final Logger logger =
LoggerFactory.getLogger(MetadataCleanupJob.class);
boolean delete = false;
private KylinConfig config = null;
-public static final long TIME_THREADSHOLD = 1 * 3600 * 1000L; // 1 hour
-public static final long TIME_THREADSHOLD_FOR_JOB = 30 * 24 * 3600 *
1000L; // 30 days
+private static final long TIME_THREADSHOLD = 1 * 3600 * 1000L; // 1 hour
+private static final int DEFAULT_DAY_THREADSHOLD_FOR_JOB = 30 ; // 30 days
/*
* (non-Javadoc)
@@ -68,10 +71,12 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
logger.info("jobs args: " + Arrays.toString(args));
options.addOption(OPTION_DELETE);
+options.addOption(OPTION_THRESHOLD_FOR_JOB);
parseOptions(options, args);
logger.info("options: '" + getOptionsAsString() + "'");
logger.info("delete option value: '" + getOptionValue(OPTION_DELETE) +
"'");
+logger.info("jobThreshold option value: '" +
getOptionValue(OPTION_THRESHOLD_FOR_JOB) + "'");
delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
config = KylinConfig.getInstanceFromEnv();
@@ -148,7 +153,10 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
for (ExecutablePO executable : allExecutable) {
long lastModified = executable.getLastModified();
ExecutableOutputPO output =
executableDao.getJobOutput(executable.getUuid());
-if (System.currentTimeMillis() - lastModified >
TIME_THREADSHOLD_FOR_JOB &&
(ExecutableState.SUCCEED.toString().equals(output.getStatus()) ||
ExecutableState.DISCARDED.toString().equals(output.getStatus( {
+int jobThresholdDay =
optionsHelper.hasOption(OPTION_THRESHOLD_FOR_JOB) ?
Integer.valueOf(optionsHelper.getOptionValue(OPTION_THRESHOLD_FOR_JOB)) :
DEFAULT_DAY_THREADSHOLD_FOR_JOB;
+long jobThresholdTime = jobThresholdDay * 24 * 3600 * 1000L;
+
+if (System.currentTimeMillis() - lastModified > jobThresholdTime
&& (ExecutableState.SUCCEED.toString().equals(output.getStatus()) ||
ExecutableState.DISCARDED.toString().equals(output.getStatus( {
toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/"
+ executable.getUuid());
toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" +
executable.getUuid());
[46/50] [abbrv] kylin git commit: fix KYLIN-2594: After reloading metadata, refresh the project list
fix KYLIN-2594: After reloading metadata, refresh the project list
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87fedf75
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87fedf75
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87fedf75
Branch: refs/heads/KYLIN-2624
Commit: 87fedf75e5bbf61a33cf1b31720ad9ac55de6287
Parents: a34db26
Author: pengjianlin
Authored: Fri May 19 16:24:28 2017 +0800
Committer: Billy Liu
Committed: Fri May 19 16:44:28 2017 +0800
--
webapp/app/js/controllers/admin.js | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/87fedf75/webapp/app/js/controllers/admin.js
--
diff --git a/webapp/app/js/controllers/admin.js
b/webapp/app/js/controllers/admin.js
index 10e2c38..37c1d67 100644
--- a/webapp/app/js/controllers/admin.js
+++ b/webapp/app/js/controllers/admin.js
@@ -18,7 +18,7 @@
'use strict';
-KylinApp.controller('AdminCtrl', function ($scope, AdminService, CacheService,
TableService, loadingRequest, MessageService, $modal,
SweetAlert,kylinConfig,ProjectModel,$window) {
+KylinApp.controller('AdminCtrl', function ($scope, AdminService, CacheService,
TableService, loadingRequest, MessageService, ProjectService, $modal,
SweetAlert,kylinConfig,ProjectModel,$window) {
$scope.configStr = "";
$scope.envStr = "";
@@ -84,7 +84,7 @@ KylinApp.controller('AdminCtrl', function ($scope,
AdminService, CacheService, T
}
});
}
-
+
$scope.reloadMeta = function () {
SweetAlert.swal({
title: '',
@@ -98,6 +98,9 @@ KylinApp.controller('AdminCtrl', function ($scope,
AdminService, CacheService, T
if (isConfirm) {
CacheService.clean({}, function () {
SweetAlert.swal('Success!', 'Cache reload successfully', 'success');
+ ProjectService.listReadable({}, function(projects) {
+ProjectModel.setProjects(projects);
+ });
}, function (e) {
if (e.data && e.data.exception) {
var message = e.data.exception;
[39/50] [abbrv] kylin git commit: KYLIN-2625 not null filter clause should be evaluable in storage
KYLIN-2625 not null filter clause should be evaluable in storage
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c9dc7cc7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c9dc7cc7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c9dc7cc7
Branch: refs/heads/KYLIN-2624
Commit: c9dc7cc78c5eed40f4c5a438423b857a145f17b1
Parents: a38b02d
Author: Hongbin Ma
Authored: Tue May 16 18:04:03 2017 +0800
Committer: Roger Shi
Committed: Tue May 16 20:59:12 2017 +0800
--
.../metadata/filter/CompareTupleFilter.java | 3 ++-
.../test/resources/query/sql_limit/query04.sql | 25
2 files changed, 27 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9dc7cc7/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index d783e52..2771250 100755
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -215,7 +215,8 @@ public class CompareTupleFilter extends TupleFilter {
@Override
public boolean isEvaluable() {
return (column != null || (function != null &&
function.isEvaluable())) //
-&& !conditionValues.isEmpty() && secondColumn == null;
+&& (!conditionValues.isEmpty() || operator ==
FilterOperatorEnum.ISNOTNULL || operator == FilterOperatorEnum.ISNULL) //
+&& secondColumn == null;
}
public boolean alwaysReturnTrue() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9dc7cc7/kylin-it/src/test/resources/query/sql_limit/query04.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_limit/query04.sql
b/kylin-it/src/test/resources/query/sql_limit/query04.sql
new file mode 100644
index 000..381ab65
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_limit/query04.sql
@@ -0,0 +1,25 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+
+select * from (
+select * from test_kylin_fact
+ where lstg_format_name is not null
+ ) limit 20
+
[16/50] [abbrv] kylin git commit: KYLIN-2598 Should not translate filter to a in-clause filter with too many elements
KYLIN-2598 Should not translate filter to a in-clause filter with too many
elements
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9dc6553d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9dc6553d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9dc6553d
Branch: refs/heads/KYLIN-2624
Commit: 9dc6553d29eb3f106823026f45dc878347a2a108
Parents: 529b1be
Author: Hongbin Ma
Authored: Tue May 9 18:33:21 2017 +0800
Committer: Dong Li
Committed: Tue May 9 20:30:21 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4 +++
.../kylin/dict/BuiltInFunctionTransformer.java | 10 +-
.../test/resources/query/sql_like/query01.sql | 6 ++--
.../test/resources/query/sql_like/query05.sql | 33
.../test/resources/query/sql_like/query06.sql | 33
.../test/resources/query/sql_like/query22.sql | 31 ++
.../test/resources/query/sql_like/query23.sql | 31 ++
7 files changed, 144 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index eafdbbb..10209d7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -844,6 +844,10 @@ abstract public class KylinConfigBase implements
Serializable {
return value > 0 ? value : Long.MAX_VALUE;
}
+public int getTranslatedInClauseMaxSize() {
+return
Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size",
"1024"));
+}
+
public int getLargeQueryThreshold() {
return
Integer.parseInt(getOptional("kylin.query.large-query-threshold",
String.valueOf(100)));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 6ef2f96..4d7b15d 100755
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict;
import java.util.Collection;
import java.util.ListIterator;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dimension.IDimensionEncodingMap;
import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
@@ -93,12 +94,19 @@ public class BuiltInFunctionTransformer implements
ITupleFilterTransformer {
translated.addChild(new ColumnTupleFilter(columnRef));
try {
+int translatedInClauseMaxSize =
KylinConfig.getInstanceFromEnv().getTranslatedInClauseMaxSize();
+
for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
Object dictVal = dict.getValueFromId(i);
if ((Boolean)
builtInFunctionTupleFilter.invokeFunction(dictVal)) {
translated.addChild(new ConstantTupleFilter(dictVal));
+
+if (translated.getChildren().size() >
translatedInClauseMaxSize) {
+return null;
+}
}
}
+
} catch (Exception e) {
logger.debug(e.getMessage());
return null;
@@ -175,4 +183,4 @@ public class BuiltInFunctionTransformer implements
ITupleFilterTransformer {
}
return translated;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query01.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_like/query01.sql
b/kylin-it/src/test/resources/query/sql_like/query01.sql
index e1f131d..e3a05ca 100644
--- a/kylin-it/src/test/resources/query/sql_like/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_like/query01.sql
@@ -16,7 +16,7 @@
-- limitations under the License.
--
-select lstg_format_name as lstg_format_name, count(*) as cnt
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt
from test_kylin_fact
inner JOIN edw.test_cal_dt as test_cal_dt
@@ -27,5 +27,5 @@ inner JOIN edw.test
[36/50] [abbrv] kylin git commit: KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in Broadcaster
KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in Broadcaster
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c3ed36f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c3ed36f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c3ed36f
Branch: refs/heads/KYLIN-2624
Commit: 1c3ed36f7dcf2152b17966bb69dd3211ce4e8aba
Parents: 1c80c29
Author: gaodayue
Authored: Sat May 6 05:21:22 2017 +0800
Committer: kangkaisen
Committed: Mon May 15 16:02:01 2017 +0800
--
.../java/org/apache/kylin/common/restclient/RestClient.java | 4
.../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 7 +--
2 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3ed36f/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 33a4e7a..470386b 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -287,4 +287,8 @@ public class RestClient {
return result.toString();
}
+@Override
+public String toString() {
+return String.format("RestClient(%s:%d)", host, port);
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3ed36f/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 1394f7b..e787dfd 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.DaemonThreadFactory;
+import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,7 +117,7 @@ public class Broadcaster {
for (String node : config.getRestServers()) {
restClients.add(new RestClient(node));
}
-final ExecutorService wipingCachePool =
Executors.newFixedThreadPool(restClients.size(), new DaemonThreadFactory());
+final ExecutorService wipingCachePool =
Executors.newCachedThreadPool(new DaemonThreadFactory());
while (true) {
try {
final BroadcastEvent broadcastEvent =
broadcastEvents.takeFirst();
@@ -125,8 +126,10 @@ public class Broadcaster {
wipingCachePool.execute(new Runnable() {
@Override
public void run() {
-try {
+try (SetThreadName ignored = new
SetThreadName("CacheWiper %s %s", restClient, broadcastEvent)) {
+logger.info("{} wipe cache {}",
restClient, broadcastEvent);
restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(),
broadcastEvent.getCacheKey());
+logger.info("{} wipe cache {}
success", restClient, broadcastEvent);
} catch (IOException e) {
logger.warn("Thread failed during wipe
cache at " + broadcastEvent, e);
}
[23/50] [abbrv] kylin git commit: KYLIN-2608 Bubble sort bug in JoinDesc
KYLIN-2608 Bubble sort bug in JoinDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a059064d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a059064d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a059064d
Branch: refs/heads/KYLIN-2624
Commit: a059064df85d26014150709273cb2535805adadf
Parents: f277b2d
Author: Hongbin Ma
Authored: Thu May 11 16:24:56 2017 +0800
Committer: liyang-gmt8
Committed: Thu May 11 16:27:38 2017 +0800
--
.../src/main/java/org/apache/kylin/metadata/model/JoinDesc.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a059064d/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index dd1500b..eb82ace 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -125,7 +125,7 @@ public class JoinDesc implements Serializable {
int n = foreignKey.length;
for (int i = 0; i < n - 1 && cont; i++) {
cont = false;
-for (int j = i; j < n - 1; j++) {
+for (int j = 0; j < n - 1 - i; j++) {
int jj = j + 1;
if (foreignKey[j].compareTo(foreignKey[jj]) > 0) {
swap(foreignKey, j, jj);
[33/50] [abbrv] kylin git commit: KYLIN-2613 fix DimensionDesc.hashCode()
KYLIN-2613 fix DimensionDesc.hashCode()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8524efe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8524efe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8524efe
Branch: refs/heads/KYLIN-2624
Commit: c8524efe7c49db6a3225ef0caa7bf900f55c2ae3
Parents: fd671d1
Author: Yang Li
Authored: Sun May 14 16:42:34 2017 +0800
Committer: Yang Li
Committed: Sun May 14 16:42:34 2017 +0800
--
.../src/main/java/org/apache/kylin/cube/model/DimensionDesc.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8524efe/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
index 344b7c8..578831f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
@@ -33,6 +33,7 @@ import com.google.common.base.Objects;
/**
*/
+@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility =
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility =
Visibility.NONE)
public class DimensionDesc implements java.io.Serializable {
@@ -178,7 +179,7 @@ public class DimensionDesc implements java.io.Serializable {
final int prime = 31;
int result = 1;
result = prime * result + ((name == null) ? 0 : name.hashCode());
-result = prime * result + ((column == null) ? 0 : name.hashCode());
+result = prime * result + ((column == null) ? 0 : column.hashCode());
result = prime * result + ((table == null) ? 0 : table.hashCode());
result = prime * result + ((derived == null) ? 0 : derived.hashCode());
return result;
[27/50] [abbrv] kylin git commit: minor, remove useless scripts
minor, remove useless scripts
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b08d278e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b08d278e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b08d278e
Branch: refs/heads/KYLIN-2624
Commit: b08d278e5ed688457758eee43aff4f479fd0fb72
Parents: 949237d
Author: Hongbin Ma
Authored: Fri May 12 12:10:21 2017 +0800
Committer: Dong Li
Committed: Fri May 12 12:46:35 2017 +0800
--
build/script/package_diag.sh | 77 -
build/script/package_libs.sh | 100 --
2 files changed, 177 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b08d278e/build/script/package_diag.sh
--
diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh
deleted file mode 100755
index 5d163e3..000
--- a/build/script/package_diag.sh
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-echo "Checking maven..."
-
-if [ -z "$(command -v mvn)" ]
-then
-echo "Please install maven first so that Kylin packaging can proceed"
-exit 1
-else
-echo "maven check passed"
-fi
-
-echo "Checking git..."
-
-if [ -z "$(command -v git)" ]
-then
-echo "Please install git first so that Kylin packaging can proceed"
-exit 1
-else
-echo "git check passed"
-fi
-
-dir=$(dirname ${0})
-cd ${dir}/../..
-version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate
-Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' `
-
-if [ "$version" == "" ];then
- echo "Failed to identify kylin version (current: `pwd`)"
- exit 1
-fi
-
-echo "kylin version: ${version}"
-
-echo "package libraries"
-mvn clean install -DskipTests || { exit 1; }
-
-echo "copy libraries"
-rm -rf build/tool
-mkdir build/tool
-cp tool/target/kylin-tool-${version}-assembly.jar
build/tool/kylin-tool-${version}.jar
-# Copied file becomes 000 for some env (e.g. my Cygwin)
-chmod 644 build/tool/kylin-tool-${version}.jar
-
-echo 'package tar.gz'
-package_name=apache-kylin-${version}-diag
-cd build/
-rm -rf ${package_name}
-mkdir ${package_name}
-mkdir ${package_name}/bin
-cp -r tool ${package_name}
-cp bin/diag.sh ${package_name}/bin/
-rm -rf tool
-find ${package_name} -type d -exec chmod 755 {} \;
-find ${package_name} -type f -exec chmod 644 {} \;
-find ${package_name} -type f -name "*.sh" -exec chmod 755 {} \;
-mkdir -p ../dist
-tar -cvzf ../dist/${package_name}.tar.gz ${package_name}
-rm -rf ${package_name}
-
-echo "Library package ready: dist/${package_name}.tar.gz"
http://git-wip-us.apache.org/repos/asf/kylin/blob/b08d278e/build/script/package_libs.sh
--
diff --git a/build/script/package_libs.sh b/build/script/package_libs.sh
deleted file mode 100755
index 7ef6b5d..000
--- a/build/script/package_libs.sh
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-echo "Checking maven..."
-
-if [ -z "$(command -v mvn)" ]
-then
-echo "Please install maven first so that Kylin packaging can proceed"
-exit 1
-else
-echo "maven check passed"
-fi
-
-echo "Checking git..."
-
-if [ -z "$
[38/50] [abbrv] kylin git commit: KYLIN-2623 Move output(Hbase) code to outputside
KYLIN-2623 Move output(Hbase) code to outputside
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a38b02df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a38b02df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a38b02df
Branch: refs/heads/KYLIN-2624
Commit: a38b02df0387541684aa1689e044927af650f1c7
Parents: edc4d4c
Author: Roger Shi
Authored: Tue May 16 13:50:23 2017 +0800
Committer: liyang-gmt8
Committed: Tue May 16 14:03:50 2017 +0800
--
.../org/apache/kylin/engine/mr/IMROutput2.java | 33 +-
.../engine/mr/common/AbstractHadoopJob.java | 6 +-
.../apache/kylin/engine/mr/steps/CuboidJob.java | 37 +++
.../kylin/engine/mr/steps/InMemCuboidJob.java | 55 ++
.../engine/mr/steps/LayerReducerNumSizing.java | 80 --
.../kylin/engine/mr/steps/MergeCuboidJob.java | 37 ---
.../engine/mr/steps/MergeCuboidMapper.java | 35 +-
.../kylin/engine/mr/steps/ReducerNumSizing.java | 106 +++
.../apache/kylin/source/hive/HiveMRInput.java | 2 -
.../apache/kylin/source/kafka/KafkaMRInput.java | 1 -
.../hbase/steps/HBaseMROutput2Transition.java | 97 +
11 files changed, 284 insertions(+), 205 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a38b02df/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
index 603f207..69bba0a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
@@ -18,11 +18,14 @@
package org.apache.kylin.engine.mr;
+import java.util.List;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import java.util.List;
-
public interface IMROutput2 {
/** Return a helper to participate in batch cubing job flow. */
@@ -53,6 +56,19 @@ public interface IMROutput2 {
/** Add step that does any necessary clean up. */
public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow);
+
+public IMROutputFormat getOuputFormat();
+
+}
+
+public interface IMROutputFormat {
+
+/** Configure the InputFormat of given job. */
+public void configureJobInput(Job job, String input) throws Exception;
+
+/** Configure the OutputFormat of given job. */
+public void configureJobOutput(Job job, String output, CubeSegment
segment, int level) throws Exception;
+
}
/** Return a helper to participate in batch merge job flow. */
@@ -82,6 +98,19 @@ public interface IMROutput2 {
/** Add step that does any necessary clean up. */
public void addStepPhase3_Cleanup(DefaultChainedExecutable jobFlow);
+
+public IMRMergeOutputFormat getOuputFormat();
+}
+
+public interface IMRMergeOutputFormat {
+
+/** Configure the InputFormat of given job. */
+public void configureJobInput(Job job, String input) throws Exception;
+
+/** Configure the OutputFormat of given job. */
+public void configureJobOutput(Job job, String output, CubeSegment
segment) throws Exception;
+
+public CubeSegment findSourceSegment(FileSplit fileSplit, CubeInstance
cube);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a38b02df/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 44686d6..764cbdd 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -559,7 +559,7 @@ public abstract class AbstractHadoopJob extends Configured
implements Tool {
HadoopUtil.deletePath(conf, path);
}
-protected double getTotalMapInputMB() throws ClassNotFoundException,
IOException, InterruptedException, JobException {
+public static double getTotalMapInputMB(Job job) throws
ClassNotFoundException, IOException, InterruptedException, JobException {
if (job == null) {
throw new JobException("Job is null");
}
@@ -576,6 +576,10 @@ public abstract class AbstractHadoopJob extends Configured
imple
[45/50] [abbrv] kylin git commit: KYLIN-2624 introduce StorageURL and set timeout for metadata access
KYLIN-2624 introduce StorageURL and set timeout for metadata access
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2dfc5bb3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2dfc5bb3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2dfc5bb3
Branch: refs/heads/KYLIN-2624
Commit: 2dfc5bb36a00245b95deeebb5bb3bca9c353c934
Parents: 7c152c5
Author: Li Yang
Authored: Thu May 18 14:16:17 2017 +0800
Committer: nichunen
Committed: Fri May 19 13:52:53 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 30 ++-
.../org/apache/kylin/common/StorageURL.java | 183 +++
.../common/persistence/FileResourceStore.java | 2 +-
.../apache/kylin/common/KylinConfigTest.java| 9 +-
.../org/apache/kylin/common/StorageURLTest.java | 103 +++
.../persistence/LocalFileResourceStoreTest.java | 3 +-
.../common/persistence/ResourceStoreTest.java | 18 +-
.../common/util/LocalFileMetadataTestCase.java | 7 +
.../apache/kylin/dict/DictionaryManager.java| 18 +-
.../dict/global/AppendTrieDictionaryTest.java | 5 +-
.../impl/threadpool/DistributedScheduler.java | 11 +-
.../apache/kylin/storage/StorageContext.java| 7 +-
.../kylin/engine/mr/steps/CubeReducerTest.java | 2 +-
.../engine/mr/steps/NDCuboidMapperTest.java | 2 +-
.../kylin/job/BaseTestDistributedScheduler.java | 5 +-
.../kylin/storage/hbase/ITStorageTest.java | 3 +-
.../apache/kylin/query/schema/OLAPSchema.java | 5 +-
.../rest/security/MockAclHBaseStorage.java | 11 +-
.../rest/security/RealAclHBaseStorage.java | 15 +-
.../apache/kylin/rest/service/QueryService.java | 11 +-
.../kylin/storage/hbase/HBaseConnection.java| 51 +++---
.../kylin/storage/hbase/HBaseResourceStore.java | 46 +++--
.../storage/hbase/util/CubeMigrationCLI.java| 20 +-
.../hbase/util/GridTableHBaseBenchmark.java | 3 +-
.../kylin/storage/hdfs/HDFSResourceStore.java | 50 ++---
.../hbase/steps/CubeHFileMapper2Test.java | 4 +-
.../storage/hbase/steps/MockupMapContext.java | 2 +-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 20 +-
28 files changed, 461 insertions(+), 185 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfc5bb3/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b620924..caef95e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -169,7 +169,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String toString() {
-return getMetadataUrl();
+return getMetadataUrl().toString();
}
//
@@ -205,8 +205,8 @@ abstract public class KylinConfigBase implements
Serializable {
// METADATA
//
-public String getMetadataUrl() {
-return getOptional("kylin.metadata.url");
+public StorageURL getMetadataUrl() {
+return StorageURL.valueOf(getOptional("kylin.metadata.url", ""));
}
// for test only
@@ -215,14 +215,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String getMetadataUrlPrefix() {
-String metadataUrl = getMetadataUrl();
-String defaultPrefix = "kylin_metadata";
-
-int cut = metadataUrl.lastIndexOf('@');
-if (cut > 0) {
-return metadataUrl.substring(0, cut);
-}
-return defaultPrefix;
+return getMetadataUrl().getIdentifier();
}
public String[] getRealizationProviders() {
@@ -558,8 +551,14 @@ abstract public class KylinConfigBase implements
Serializable {
return Integer.parseInt(getOptional("kylin.storage.default", "2"));
}
-public String getStorageUrl() {
-return getOptional("kylin.storage.url");
+public StorageURL getStorageUrl() {
+String url = getOptional("kylin.storage.url", "default@hbase");
+
+// for backward compatibility
+if ("hbase".equals(url))
+url = "default@hbase";
+
+return StorageURL.valueOf(url);
}
public String getHBaseClusterFs() {
@@ -987,10 +986,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String getResourceStoreImpl() {
-String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
-
[40/50] [abbrv] kylin git commit: KYLIN-2621 The user of the LDAP group named admin always has ROLE_ADMIN permission
KYLIN-2621 The user of the LDAP group named admin always has ROLE_ADMIN
permission
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a34db26f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a34db26f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a34db26f
Branch: refs/heads/KYLIN-2624
Commit: a34db26f0bd594b55fd4df29b66f15579bb3c8ff
Parents: 858d947
Author: 10069681
Authored: Mon May 15 20:31:26 2017 +0800
Committer: Billy Liu
Committed: Wed May 17 16:23:12 2017 +0800
--
.../rest/security/AuthoritiesPopulator.java | 20 ++--
1 file changed, 10 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a34db26f/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
index 2b290ce..592791c 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
@@ -52,8 +52,11 @@ public class AuthoritiesPopulator extends
DefaultLdapAuthoritiesPopulator {
this.adminRoleAsAuthority = new SimpleGrantedAuthority(adminRole);
String[] defaultRoles = StringUtils.split(defaultRole, ",");
-if (ArrayUtils.contains(defaultRoles, Constant.ROLE_MODELER))
+if (ArrayUtils.contains(defaultRoles, Constant.ROLE_MODELER)) {
this.defaultAuthorities.add(modelerAuthority);
+this.defaultAuthorities.add(analystAuthority);
+}
+
if (ArrayUtils.contains(defaultRoles, Constant.ROLE_ANALYST))
this.defaultAuthorities.add(analystAuthority);
}
@@ -62,19 +65,16 @@ public class AuthoritiesPopulator extends
DefaultLdapAuthoritiesPopulator {
public Set getGroupMembershipRoles(String userDn, String
username) {
Set authorities =
super.getGroupMembershipRoles(userDn, username);
-authorities.addAll(defaultAuthorities);
+Set userAuthorities = new
HashSet();
+userAuthorities.addAll(defaultAuthorities);
if (authorities.contains(adminRoleAsAuthority)) {
-authorities.add(adminAuthority);
-authorities.add(modelerAuthority);
-authorities.add(analystAuthority);
-}
-
-if (authorities.contains(modelerAuthority)) {
-authorities.add(analystAuthority);
+userAuthorities.add(adminAuthority);
+userAuthorities.add(modelerAuthority);
+userAuthorities.add(analystAuthority);
}
-return authorities;
+return userAuthorities;
}
}
[50/50] [abbrv] kylin git commit: KYLIN-2624 fix again
KYLIN-2624 fix again
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3c9607fc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3c9607fc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3c9607fc
Branch: refs/heads/KYLIN-2624
Commit: 3c9607fc48508c2ea8573cfa84c215f9adcac8e4
Parents: be32f75
Author: Yang Li
Authored: Sat May 20 07:56:24 2017 +0800
Committer: Yang Li
Committed: Sat May 20 07:56:24 2017 +0800
--
.../apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/3c9607fc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
index dc2a23e..05d3a5a 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
@@ -102,7 +102,7 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
public Factory(KylinConfig config) {
this.curator = getZKClient(config);
-this.zkPathBase = "/kylin/" +
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
+this.zkPathBase = dropDoubleSlash("/kylin/" +
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix());
}
@Override
[35/50] [abbrv] kylin git commit: KYLIN-2557 code review
KYLIN-2557 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c80c29b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c80c29b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c80c29b
Branch: refs/heads/KYLIN-2624
Commit: 1c80c29b22d9c00de1eb2e9a09c21377714248b7
Parents: e6a8a00
Author: Yang Li
Authored: Sun May 14 20:14:29 2017 +0800
Committer: Yang Li
Committed: Sun May 14 20:14:29 2017 +0800
--
.../kylin/storage/hbase/HBaseConnection.java| 54 ++--
1 file changed, 27 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c80c29b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index ef82641..5fafa2b 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -18,7 +18,18 @@
package org.apache.kylin.storage.hbase;
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -39,17 +50,7 @@ import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Sets;
/**
* @author yangli9
@@ -67,8 +68,6 @@ public class HBaseConnection {
private static ExecutorService coprocessorPool = null;
-private static DistributedLock lock = null;
-
static {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
@@ -268,18 +267,20 @@ public class HBaseConnection {
}
public static void createHTableIfNeeded(Connection conn, String table,
String... families) throws IOException {
-Admin hbase = conn.getAdmin();
+Admin admin = conn.getAdmin();
TableName tableName = TableName.valueOf(table);
-boolean hasLock = false;
+DistributedLock lock = null;
+String lockPath = getLockPath(table);
+
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
-Set existingFamilies =
getFamilyNames(hbase.getTableDescriptor(tableName));
+Set existingFamilies =
getFamilyNames(admin.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable
'" + table + "'");
-hbase.addColumn(tableName,
newFamilyDescriptor(family));
+admin.addColumn(tableName,
newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it
finish?
wait = true;
}
@@ -295,7 +296,8 @@ public class HBaseConnection {
}
lock =
KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentProcess();
-hasLock = lock.lock(getLockPath(table), Long.MAX_VALUE);
+if (!lock.lock(lockPath, Long.MAX_VALUE))
+throw new RuntimeException("Cannot acquire lock to create
HTable " + table);
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
@@ -313,15 +315,13 @@ public class HBaseConnection {
}
}
-//desc.setValue(HTABLE_UUID_TAG, UUID.randomUUID().toString());
-hbase.createTable(desc);
+admin.createTable(desc);
logger.debug
[43/50] [abbrv] kylin git commit: KYLIN-2577, refine hive configurations during cubing steps
KYLIN-2577, refine hive configurations during cubing steps
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c152c51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c152c51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c152c51
Branch: refs/heads/KYLIN-2624
Commit: 7c152c515ce6cebe380c4e1b7c16d109192308db
Parents: 0061900
Author: Cheng Wang
Authored: Thu May 18 11:05:34 2017 +0800
Committer: liyang-gmt8
Committed: Thu May 18 15:27:36 2017 +0800
--
build/bin/load-hive-conf.sh | 2 +-
.../kylin/common/util/HiveCmdBuilder.java | 56 +++-
.../kylin/common/util/HiveCmdBuilderTest.java | 2 +-
.../kylin/job/engine/JobEngineConfig.java | 43 ---
.../source/hive/CreateFlatHiveTableStep.java| 3 --
.../apache/kylin/source/hive/HiveMRInput.java | 4 --
6 files changed, 57 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/build/bin/load-hive-conf.sh
--
diff --git a/build/bin/load-hive-conf.sh b/build/bin/load-hive-conf.sh
index a5046f0..a42cd0c 100644
--- a/build/bin/load-hive-conf.sh
+++ b/build/bin/load-hive-conf.sh
@@ -11,5 +11,5 @@ hive_conf_prop="${KYLIN_HOME}/logs/hive_props"
rm -rf ${hive_conf_prop}
export ENABLE_CHECK_ENV=false
${dir}/kylin.sh org.apache.kylin.tool.HiveConfigCLI ${hive_conf_dir}
${hive_conf_prop}
-[[ 0 == $? ]] || quit "Error, can not parse ${hive_conf_dir} and can not apply
it to hive relevant check."
+[[ 0 == $? ]] || quit "Can not parse xml file: ${hive_conf_dir}, please check
it."
hive_conf_properties=`cat ${hive_conf_prop}`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 255867a..2f6b9a0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -27,17 +27,25 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
import com.google.common.collect.Lists;
public class HiveCmdBuilder {
private static final Logger logger =
LoggerFactory.getLogger(HiveCmdBuilder.class);
+public static final String HIVE_CONF_FILENAME = "kylin_hive_conf";
+
public enum HiveClientMode {
CLI, BEELINE
}
@@ -50,6 +58,7 @@ public class HiveCmdBuilder {
public HiveCmdBuilder() {
kylinConfig = KylinConfig.getInstanceFromEnv();
clientMode =
HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase());
+loadHiveConfiguration();
}
public String build() {
@@ -75,8 +84,8 @@ public class HiveCmdBuilder {
bw.newLine();
}
buf.append("beeline ");
-buf.append(parseProps());
buf.append(kylinConfig.getHiveBeelineParams());
+buf.append(parseProps());
buf.append(" -f ");
buf.append(tmpHql.getAbsolutePath());
buf.append(";ret_code=$?;rm -f ");
@@ -123,6 +132,7 @@ public class HiveCmdBuilder {
}
public void setHiveConfProps(Map hiveConfProps) {
+this.hiveConfProps.clear();
this.hiveConfProps.putAll(hiveConfProps);
}
@@ -144,4 +154,48 @@ public class HiveCmdBuilder {
public String toString() {
return build();
}
+
+private void loadHiveConfiguration() {
+
+File hiveConfFile;
+String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml");
+String path = System.getProperty(KylinConfig.KYLIN_CONF);
+
+if (StringUtils.isNotEmpty(path)) {
+hiveConfFile = new File(path, hiveConfFileName);
+} else {
+path = KylinConfig.getKylinHome();
+if (StringUtils.isEmpty(path)) {
+logger.error("KYLIN_HOME is not set, can not locate hive conf:
{}.xml", HIVE_CONF_FILENAME);
+return;
+}
+hiveConfFile = new File(path + File.separator + "conf",
hiveConfF
[30/50] [abbrv] kylin git commit: KYLIN-2596 Enable generating multiple streaming messages with one input message in streaming parser
KYLIN-2596 Enable generating multiple streaming messages with one input message
in streaming parser
* Minor, remove useless imports.
* Enable generating multiple streaming messages with one input message in
streaming parser
* Make MR input can generate multiple rows of date.
* For multiple rows, outputKV() should be called for each row.
* Try&catch for each row's data processing.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/edc4d4cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/edc4d4cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/edc4d4cc
Branch: refs/heads/KYLIN-2624
Commit: edc4d4cc558473476b18d48f232635e44640c27a
Parents: 7c0038d
Author: nichunen
Authored: Fri May 12 14:55:14 2017 +0800
Committer: hongbin ma
Committed: Fri May 12 14:55:14 2017 +0800
--
.../java/org/apache/kylin/job/DeployUtil.java | 2 +-
.../kylin/common/util/StreamingMessage.java | 62 --
.../kylin/common/util/StreamingMessageRow.java | 62 ++
.../org/apache/kylin/engine/mr/IMRInput.java| 4 +-
.../mr/steps/FactDistinctColumnsMapper.java | 113 ++-
.../engine/mr/steps/HiveToBaseCuboidMapper.java | 15 ++-
.../engine/mr/steps/InMemCuboidMapper.java | 15 ++-
.../apache/kylin/source/hive/HiveMRInput.java | 6 +-
.../cardinality/ColumnCardinalityMapper.java| 25 ++--
.../apache/kylin/source/kafka/KafkaMRInput.java | 16 ++-
.../kylin/source/kafka/StreamingParser.java | 8 +-
.../source/kafka/TimedJsonStreamParser.java | 11 +-
.../source/kafka/TimedJsonStreamParserTest.java | 22 ++--
13 files changed, 194 insertions(+), 167 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/edc4d4cc/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index fdcd52c..077c056 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -166,7 +166,7 @@ public class DeployUtil {
TimedJsonStreamParser timedJsonStreamParser = new
TimedJsonStreamParser(tableColumns, null);
StringBuilder sb = new StringBuilder();
for (String json : data) {
-List rowColumns =
timedJsonStreamParser.parse(ByteBuffer.wrap(json.getBytes())).getData();
+List rowColumns =
timedJsonStreamParser.parse(ByteBuffer.wrap(json.getBytes())).get(0).getData();
sb.append(StringUtils.join(rowColumns, ","));
sb.append(System.getProperty("line.separator"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/edc4d4cc/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
deleted file mode 100644
index 981c8a8..000
---
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- */
-public class StreamingMessage {
-
-private final List data;
-
-private long offset;
-
-private long timestamp;
-
-private Map params;
-
-public StreamingMessage(List data, long offset, long timestamp,
Map params) {
-this.data = data;
-this.offset = offset;
-this.timestamp = timestamp;
-this.params = params;
-}
-
-public final List getData() {
-return data;
-}
-
-public final long getOffset() {
-return offset;
-}
-
-public void setOffset(long offset) {
-this.offset = offset;
-}
-
-
[49/50] [abbrv] kylin git commit: KYLIN-2624 fix IT with refactored DistributedLock
KYLIN-2624 fix IT with refactored DistributedLock
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be32f758
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be32f758
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be32f758
Branch: refs/heads/KYLIN-2624
Commit: be32f75883c21bf32545d1b06158ad7cf36d676b
Parents: f36372b
Author: Yang Li
Authored: Fri May 19 22:02:38 2017 +0800
Committer: Yang Li
Committed: Fri May 19 22:02:38 2017 +0800
--
.../job/impl/threadpool/DistributedScheduler.java | 1 +
.../hbase/util/ZookeeperDistributedLock.java| 16
2 files changed, 13 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/be32f758/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index d01cb5a..8812dad 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -319,6 +319,7 @@ public class DistributedScheduler implements
Scheduler, Conn
}
return path;
}
+
@Override
public void shutdown() throws SchedulerException {
logger.info("Will shut down Job Engine ");
http://git-wip-us.apache.org/repos/asf/kylin/blob/be32f758/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
index d181d81..dc2a23e 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
@@ -281,10 +281,18 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
// normalize lock path
private String norm(String lockPath) {
-if (lockPath.startsWith(zkPathBase))
-return lockPath;
-else
-return zkPathBase + (lockPath.startsWith("/") ? "" : "/") +
lockPath;
+if (!lockPath.startsWith(zkPathBase))
+lockPath = zkPathBase + (lockPath.startsWith("/") ? "" : "/") +
lockPath;
+
+return dropDoubleSlash(lockPath);
+}
+
+public static String dropDoubleSlash(String path) {
+for (int n = Integer.MAX_VALUE; n > path.length();) {
+n = path.length();
+path = path.replace("//", "/");
+}
+return path;
}
//
[19/50] [abbrv] kylin git commit: KYLIN-2599 more complete fix
KYLIN-2599 more complete fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f277b2d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f277b2d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f277b2d5
Branch: refs/heads/KYLIN-2624
Commit: f277b2d586d4b306d15a2fc7de83b1dfb38d628e
Parents: c3d8183
Author: Hongbin Ma
Authored: Wed May 10 13:52:14 2017 +0800
Committer: liyang-gmt8
Committed: Wed May 10 14:16:18 2017 +0800
--
.../calcite/sql2rel/SqlToRelConverter.java | 43 ++--
.../apache/kylin/query/ITKylinQueryTest.java| 1 +
.../resources/query/sql_verifyCount/query09.sql | 34
.../query/sql_verifyCount/query09.sql.expected | 2 +
.../resources/query/sql_verifyCount/query10.sql | 34
.../query/sql_verifyCount/query10.sql.expected | 2 +
.../resources/query/sql_verifyCount/query11.sql | 36
.../query/sql_verifyCount/query11.sql.expected | 2 +
8 files changed, 142 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f277b2d5/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index a520aa6..8085a70 100644
---
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -29,6 +29,7 @@ import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.prepare.RelOptTableImpl;
import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationImpl;
import org.apache.calcite.rel.RelCollationTraitDef;
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelFieldCollation;
@@ -613,15 +614,6 @@ public class SqlToRelConverter {
return root;
}
-RelNode tableScanOrJoin = null;
-RelNode rootProjInput = rootPrj.getInput(0) instanceof LogicalFilter ?
rootPrj.getInput(0).getInput(0): rootPrj.getInput(0);
-if(rootProjInput instanceof TableScan || rootProjInput instanceof
LogicalJoin) {
-tableScanOrJoin = rootProjInput;
-}
-if(tableScanOrJoin == null) {
-return root;
-}
-
//
RelNode input = rootPrj.getInput();
//if (!(//
@@ -637,25 +629,52 @@ public class SqlToRelConverter {
List inFields = inType.getFieldNames();
List projExp = new ArrayList<>();
List> projFields = new ArrayList<>();
+Map projFieldMapping = new HashMap<>();
RelDataTypeFactory.FieldInfoBuilder projTypeBuilder =
getCluster().getTypeFactory().builder();
RelDataTypeFactory.FieldInfoBuilder validTypeBuilder =
getCluster().getTypeFactory().builder();
+
+boolean hiddenColumnExists = false;
+for (int i = 0; i < root.validatedRowType.getFieldList().size(); i++) {
+if
(root.validatedRowType.getFieldNames().get(i).startsWith("_KY_"))
+hiddenColumnExists = true;
+}
+if(!hiddenColumnExists) {
+return root;
+}
+
for (int i = 0; i < inFields.size(); i++) {
if (!inFields.get(i).startsWith("_KY_")) {
projExp.add(rootPrj.getProjects().get(i));
+projFieldMapping.put(i, projFields.size());
projFields.add(Pair.of(projFields.size(), inFields.get(i)));
projTypeBuilder.add(inType.getFieldList().get(i));
-
validTypeBuilder.add(root.validatedRowType.getFieldList().get(i));
+
+if (i < root.validatedRowType.getFieldList().size()) //for
cases like kylin-it/src/test/resources/query/sql_verifyCount/query10.sql
+
validTypeBuilder.add(root.validatedRowType.getFieldList().get(i));
}
}
RelDataType projRowType =
getCluster().getTypeFactory().createStructType(projTypeBuilder);
rootPrj = LogicalProject.create(input, projExp, projRowType);
if (rootSort != null) {
-rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(),
rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch);
+//for cases like
kylin-it/src/test/resources/query/sql_verifyCount/query10.sql, original
RelCollation is stale, need to fix its fieldIndex
+RelCollation originalCollation = rootSort.collation;
+RelCollation newCollation = null;
+List fieldCollations =
originalC
[12/50] [abbrv] kylin git commit: KYLIN-2575 Experimental feature: Computed Column
KYLIN-2575 Experimental feature: Computed Column
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a812d5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a812d5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a812d5f
Branch: refs/heads/KYLIN-2624
Commit: 9a812d5fc917ddcdb736cd1c91fdfe9c933e603a
Parents: d6fdda2
Author: Hongbin Ma
Authored: Mon May 8 15:49:57 2017 +0800
Committer: liyang-gmt8
Committed: Mon May 8 16:01:48 2017 +0800
--
.../kylin/cube/CubeCapabilityChecker.java | 11 +-
.../java/org/apache/kylin/cube/JoinChecker.java | 60 -
.../org/apache/kylin/cube/cuboid/CuboidCLI.java |7 +
.../org/apache/kylin/cube/model/CubeDesc.java | 12 +-
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 23 +
.../model/validation/rule/FunctionRuleTest.java | 11 +-
.../org/apache/kylin/job/JoinedFlatTable.java |4 +-
.../apache/kylin/metadata/MetadataManager.java | 31 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 41 +-
.../metadata/model/ComputedColumnDesc.java | 95 ++
.../kylin/metadata/model/DataModelDesc.java | 81 +-
.../apache/kylin/metadata/model/JoinsTree.java |2 +-
.../kylin/metadata/model/PartitionDesc.java | 10 +-
.../apache/kylin/metadata/model/TableDesc.java | 35 +-
.../apache/kylin/metadata/model/TableRef.java | 14 +-
.../apache/kylin/metadata/model/TblColRef.java | 36 +-
.../kylin/metadata/project/ProjectL2Cache.java | 13 +-
.../kylin/metadata/model/DataModelDescTest.java | 15 +-
.../localmeta/cube_desc/ci_inner_join_cube.json | 1016 ++---
.../localmeta/cube_desc/ci_left_join_cube.json | 1036 ++
.../model_desc/ci_inner_join_model.json | 37 +-
.../model_desc/ci_left_join_model.json | 37 +-
.../test_case_data/sandbox/kylin.properties |2 +-
.../kylin/query/CompareQueryBySuffix.java | 37 +
.../kylin/query/ICompareQueryTranslator.java| 29 +
.../apache/kylin/query/ITKylinQueryTest.java|5 +
.../org/apache/kylin/query/KylinTestBase.java | 25 +-
.../query/sql_computedcolumn/query01.sql| 14 +
.../sql_computedcolumn/query01.sql.compare | 14 +
.../query/sql_computedcolumn/query02.sql| 21 +
.../sql_computedcolumn/query02.sql.compare | 21 +
.../query/sql_verifyCount/query01.sql.expected |2 +-
.../query/sql_verifyCount/query03.sql.expected |2 +-
.../query/sql_verifyCount/query04.sql.expected |2 +-
.../apache/kylin/query/schema/OLAPSchema.java |2 +-
.../kylin/rest/controller/ModelController.java |2 +-
.../kylin/rest/service/CacheServiceTest.java|1 -
.../kylin/rest/service/ModelServiceTest.java| 97 ++
.../apache/kylin/source/hive/SchemaChecker.java |2 +-
39 files changed, 1894 insertions(+), 1011 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index c45144b..20cb0a9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -52,12 +52,7 @@ public class CubeCapabilityChecker {
CapabilityResult result = new CapabilityResult();
result.capable = false;
-// match joins
-boolean isJoinMatch = JoinChecker.isJoinMatch(digest.joinDescs, cube);
-if (!isJoinMatch) {
-logger.info("Exclude cube " + cube.getName() + " because unmatched
joins");
-return result;
-}
+// match joins is ensured at model select
// dimensions & measures
Collection dimensionColumns = getDimensionColumns(digest);
@@ -106,12 +101,12 @@ public class CubeCapabilityChecker {
}
if (!unmatchedDimensions.isEmpty()) {
-logger.info("Exclude cube " + cube.getName() + " because unmatched
dimensions");
+logger.info("Exclude cube " + cube.getName() + " because unmatched
dimensions: " + unmatchedDimensions);
return result;
}
if (!unmatchedAggregations.isEmpty()) {
-logger.info("Exclude cube " + cube.getName() + " because unmatched
aggregations");
+logger.info("Exclude cube " + cube.getName() + " because unmatched
aggregations: " + unmatchedAggregations);
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
-
[48/50] [abbrv] kylin git commit: Merge commit '87fedf75e5bbf61a33cf1b31720ad9ac55de6287'
Merge commit '87fedf75e5bbf61a33cf1b31720ad9ac55de6287'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f36372bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f36372bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f36372bc
Branch: refs/heads/KYLIN-2624
Commit: f36372bc657b1edb312b36a071b077cde184eab8
Parents: 8ce972f 87fedf7
Author: Hongbin Ma
Authored: Fri May 19 17:55:26 2017 +0800
Committer: Hongbin Ma
Committed: Fri May 19 17:55:26 2017 +0800
--
.../rel/rules/OLAPJoinPushThroughJoinRule.java | 172
.../rel/rules/OLAPJoinPushThroughJoinRule2.java | 205 +
.../java/org/apache/calcite/tools/Programs.java | 438 +++
build/bin/find-hive-dependency.sh | 4 +-
build/bin/kylin.sh | 41 +-
build/bin/sample.sh | 24 +-
.../apache/kylin/common/KylinConfigBase.java| 8 +-
.../kylin/common/lock/DistributedLock.java | 63 ++-
.../common/lock/DistributedLockFactory.java | 43 ++
.../kylin/common/restclient/RestClient.java | 16 +-
.../org/apache/kylin/common/util/CheckUtil.java | 50 ++-
.../apache/kylin/cube/model/DimensionDesc.java | 3 +-
.../kylin/dict/BuiltInFunctionTransformer.java | 8 +-
.../kylin/dict/GlobalDictionaryBuilder.java | 71 +--
.../impl/threadpool/DistributedScheduler.java | 55 +--
.../kylin/job/lock/DistributedJobLock.java | 24 -
.../kylin/metadata/cachesync/Broadcaster.java | 66 ++-
.../metadata/filter/function/BuiltInMethod.java | 39 +-
.../metadata/filter/function/LikeMatchers.java | 141 ++
examples/sample_cube/create_sample_tables.sql | 10 +-
.../kylin/job/BaseTestDistributedScheduler.java | 35 +-
.../job/ITDistributedSchedulerBaseTest.java | 6 +-
.../job/ITDistributedSchedulerTakeOverTest.java | 2 +-
.../resources/query/sql_subquery/query17.sql| 17 +
.../resources/query/sql_subquery/query18.sql| 23 +
.../resources/query/sql_subquery/query19.sql| 20 +
.../resources/query/sql_subquery/query21.sql| 26 ++
.../resources/query/sql_subquery/query22.sql| 25 ++
.../resources/query/sql_subquery/query23.sql| 26 ++
.../resources/query/sql_subquery/query24.sql| 29 ++
.../resources/query/sql_subquery/query25.sql| 22 +
.../resources/query/sql_subquery/query26.sql| 16 +
.../resources/query/sql_subquery/query27.sql| 20 +
.../resources/query/sql_subquery/query28.sql| 21 +
.../query/sql_subquery/query29.sql.todo | 22 +
.../kylin/query/relnode/OLAPAggregateRel.java | 4 +-
.../apache/kylin/query/relnode/OLAPContext.java | 10 +
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 12 +-
.../kylin/query/relnode/OLAPProjectRel.java | 15 +-
.../rest/security/AuthoritiesPopulator.java | 20 +-
.../apache/kylin/rest/service/AclService.java | 1 -
.../kylin/rest/service/CacheServiceTest.java| 10 +-
.../kylin/storage/hbase/HBaseConnection.java| 30 +-
.../hbase/util/ZookeeperDistributedJobLock.java | 257 ---
.../hbase/util/ZookeeperDistributedLock.java| 307 +
.../storage/hbase/util/ZookeeperJobLock.java| 144 ++
.../apache/kylin/storage/hdfs/LockManager.java | 8 +-
.../util/ITZookeeperDistributedLockTest.java| 254 +++
.../apache/kylin/tool/MetadataCleanupJob.java | 14 +-
webapp/app/js/controllers/access.js | 1 +
webapp/app/js/controllers/admin.js | 12 +-
webapp/app/js/controllers/cubeMeasures.js | 2 +-
webapp/app/js/controllers/query.js | 4 +-
54 files changed, 2273 insertions(+), 625 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/build/bin/find-hive-dependency.sh
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/build/bin/sample.sh
--
diff --cc build/bin/sample.sh
index 2e13805,10900bb..b9c4ea3
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@@ -52,10 -62,10 +63,11 @@@ the
else
beeline_params=${beeline_params/${hive2_url}/${hive2_url}${sample_database}}
fi
- beeline ${hive_conf_properties} ${beeline_params} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
-beeline ${beeline_params} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
++
++beeline ${hive_conf_properties} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir}
${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || {
exit 1; }
else
-hive -e "CREATE DATABASE IF NOT EX
[26/50] [abbrv] kylin git commit: minor, fix csv export garbled issue
minor, fix csv export garbled issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/949237dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/949237dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/949237dd
Branch: refs/heads/KYLIN-2624
Commit: 949237ddca0ab24bf8121ccc002d802df678761c
Parents: a059064
Author: Hongbin Ma
Authored: Thu May 11 17:29:14 2017 +0800
Committer: Dong Li
Committed: Fri May 12 12:46:35 2017 +0800
--
.../java/org/apache/kylin/rest/controller/QueryController.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/949237dd/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index c5f896d..99447a1 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -103,7 +103,7 @@ public class QueryController extends BasicController {
@ResponseBody
public void downloadQueryResult(@PathVariable String format, SQLRequest
sqlRequest, HttpServletResponse response) {
SQLResponse result = queryService.doQueryWithCache(sqlRequest);
-response.setContentType("text/" + format + ";charset=utf-8");
+response.setContentType("text/" + format + ";charset=ansi");
response.setHeader("Content-Disposition", "attachment;
filename=\"result." + format + "\"");
ICsvListWriter csvWriter = null;
[11/50] [abbrv] kylin git commit: KYLIN-2575 Experimental feature: Computed Column
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
--
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
index 221ffb6..cbf7a76 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
@@ -1,455 +1,627 @@
{
- "uuid" : "629ab7a8-3929-4dff-b59d-2100aadccd1a",
- "name" : "ci_left_join_cube",
- "model_name" : "ci_left_join_model",
- "description" : null,
- "dimensions" : [ {
-"name" : "CAL_DT",
-"table" : "TEST_CAL_DT",
-"column" : "{FK}",
-"derived" : [ "WEEK_BEG_DT" ]
- }, {
-"name" : "ORDER_ID",
-"table" : "TEST_KYLIN_FACT",
-"column" : "ORDER_ID"
- }, {
-"name" : "TEST_DATE_ENC",
-"table" : "TEST_ORDER",
-"column" : "TEST_DATE_ENC"
- }, {
-"name" : "TEST_TIME_ENC",
-"table" : "TEST_ORDER",
-"column" : "TEST_TIME_ENC"
- }, {
-"name" : "CATEGORY",
-"table" : "TEST_CATEGORY_GROUPINGS",
-"column" : "{FK}",
-"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE",
"UPD_USER" ]
- }, {
-"name" : "CATEGORY_HIERARCHY",
-"table" : "TEST_CATEGORY_GROUPINGS",
-"column" : "META_CATEG_NAME",
-"derived" : null
- }, {
-"name" : "CATEGORY_HIERARCHY",
-"table" : "TEST_CATEGORY_GROUPINGS",
-"column" : "CATEG_LVL2_NAME",
-"derived" : null
- }, {
-"name" : "CATEGORY_HIERARCHY",
-"table" : "TEST_CATEGORY_GROUPINGS",
-"column" : "CATEG_LVL3_NAME",
-"derived" : null
- }, {
-"name" : "LSTG_FORMAT_NAME",
-"table" : "TEST_KYLIN_FACT",
-"column" : "LSTG_FORMAT_NAME",
-"derived" : null
- }, {
-"name" : "SITE_ID",
-"table" : "TEST_SITES",
-"column" : "{FK}",
-"derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
-"name" : "SELLER_TYPE_CD",
-"table" : "TEST_SELLER_TYPE_DIM",
-"column" : "{FK}",
-"derived" : [ "SELLER_TYPE_DESC" ]
- }, {
-"name" : "SELLER_ID",
-"table" : "TEST_KYLIN_FACT",
-"column" : "SELLER_ID"
- }, {
-"name" : "SELLER_BUYER_LEVEL",
-"table" : "SELLER_ACCOUNT",
-"column" : "ACCOUNT_BUYER_LEVEL"
- }, {
-"name" : "SELLER_SELLER_LEVEL",
-"table" : "SELLER_ACCOUNT",
-"column" : "ACCOUNT_SELLER_LEVEL"
- }, {
-"name" : "SELLER_COUNTRY",
-"table" : "SELLER_ACCOUNT",
-"column" : "ACCOUNT_COUNTRY"
- }, {
-"name" : "SELLER_COUNTRY_NAME",
-"table" : "SELLER_COUNTRY",
-"column" : "NAME"
- }, {
-"name" : "BUYER_ID",
-"table" : "TEST_ORDER",
-"column" : "BUYER_ID"
- }, {
-"name" : "BUYER_BUYER_LEVEL",
-"table" : "BUYER_ACCOUNT",
-"column" : "ACCOUNT_BUYER_LEVEL"
- }, {
-"name" : "BUYER_SELLER_LEVEL",
-"table" : "BUYER_ACCOUNT",
-"column" : "ACCOUNT_SELLER_LEVEL"
- }, {
-"name" : "BUYER_COUNTRY",
-"table" : "BUYER_ACCOUNT",
-"column" : "ACCOUNT_COUNTRY"
- }, {
-"name" : "BUYER_COUNTRY_NAME",
-"table" : "BUYER_COUNTRY",
-"column" : "NAME"
- } ],
- "measures" : [ {
-"name" : "TRANS_CNT",
-"function" : {
- "expression" : "COUNT",
- "parameter" : {
-"type" : "constant",
-"value" : "1"
- },
- "returntype" : "bigint"
+ "uuid": "629ab7a8-3929-4dff-b59d-2100aadccd1a",
+ "name": "ci_left_join_cube",
+ "model_name": "ci_left_join_model",
+ "description": null,
+ "dimensions": [
+{
+ "name": "CAL_DT",
+ "table": "TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+"WEEK_BEG_DT"
+ ]
+},
+{
+ "name": "ORDER_ID",
+ "table": "TEST_KYLIN_FACT",
+ "column": "ORDER_ID"
+},
+{
+ "name": "TEST_DATE_ENC",
+ "table": "TEST_ORDER",
+ "column": "TEST_DATE_ENC"
+},
+{
+ "name": "TEST_TIME_ENC",
+ "table": "TEST_ORDER",
+ "column": "TEST_TIME_ENC"
+},
+{
+ "name": "CATEGORY",
+ "table": "TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+"USER_DEFINED_FIELD1",
+"USER_DEFINED_FIELD3",
+"UPD_DATE",
+"UPD_USER"
+ ]
+},
+{
+ "name": "CATEGORY_HIERARCHY",
+ "table": "TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+},
+{
+ "name": "CATEGORY_HIERARCHY",
+ "table": "TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+},
+{
+ "name": "CATEGORY_HIERARCHY",
+ "table": "TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
+},
+{
+ "name": "LSTG_FORMAT_NAME",
+ "table": "TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+},
+{
+ "name": "SITE_ID",
+ "table": "TEST_SITES",
+ "col
[07/50] [abbrv] kylin git commit: KYLIN-2578 ensures zk path is prefix-ed with /kylin/metadata-url
KYLIN-2578 ensures zk path is prefix-ed with /kylin/metadata-url
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/263a1e88
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/263a1e88
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/263a1e88
Branch: refs/heads/KYLIN-2624
Commit: 263a1e8889ba223883ba42178debeee0bd02c240
Parents: d738544
Author: Yang Li
Authored: Sun May 7 15:56:43 2017 +0800
Committer: Yang Li
Committed: Sun May 7 15:56:43 2017 +0800
--
.../kylin/common/lock/DistributedLock.java | 2 ++
.../kylin/dict/GlobalDictionaryBuilder.java | 4 +--
.../impl/threadpool/DistributedScheduler.java | 6 ++--
.../hbase/util/ZookeeperDistributedLock.java| 35 +---
.../util/ITZookeeperDistributedLockTest.java| 2 +-
5 files changed, 38 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/263a1e88/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
b/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
index e8844fd..8f1fae0 100644
---
a/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
+++
b/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
@@ -23,6 +23,8 @@ import java.util.concurrent.Executor;
/**
* A distributed lock. Every instance is owned by a client, on whose behalf
locks are acquired and/or released.
+ *
+ * Implementation must ensure all lockPath will be prefix-ed with
"/kylin/metadata-prefix" automatically.
*/
public interface DistributedLock {
http://git-wip-us.apache.org/repos/asf/kylin/blob/263a1e88/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
index 0ec7730..a593371 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
@@ -93,10 +93,8 @@ public class GlobalDictionaryBuilder implements
IDictionaryBuilder {
return new AppendTrieDictionary<>();
}
-private static final String GLOBAL_DICT_LOCK_PATH = "/kylin/dict/lock";
-
private String getLockPath(String pathName) {
-return GLOBAL_DICT_LOCK_PATH + "/" +
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix() + "/" + pathName +
"/lock";
+return "/dict/" + pathName + "/lock";
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/263a1e88/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index e80f485..0714d90 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -84,7 +84,7 @@ public class DistributedScheduler implements
Scheduler, Conn
private final static String SEGMENT_ID = "segmentId";
-public static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
+public static final String ZOOKEEPER_LOCK_PATH = "/job_engine/lock"; //
note ZookeeperDistributedLock will ensure zk path prefix: /kylin/metadata
//only for it test
public static DistributedScheduler getInstance(KylinConfig config) {
@@ -305,11 +305,11 @@ public class DistributedScheduler implements
Scheduler, Conn
}
public String getLockPath(String pathName) {
-return ZOOKEEPER_LOCK_PATH + "/" +
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix() + "/" + pathName;
+return ZOOKEEPER_LOCK_PATH + "/" + pathName;
}
private String getWatchPath() {
-return ZOOKEEPER_LOCK_PATH + "/" +
KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
+return ZOOKEEPER_LOCK_PATH;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/263a1e88/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
b/storage-hbase/src/main/java/o
[04/50] [abbrv] kylin git commit: KYLIN-2588 fix ParameterDesc.equals()
KYLIN-2588 fix ParameterDesc.equals()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03969236
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03969236
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03969236
Branch: refs/heads/KYLIN-2624
Commit: 0396923651cda8428b3022febcf013ff6d0457cf
Parents: 9a0598e
Author: lidongsjtu
Authored: Fri May 5 18:49:48 2017 +0800
Committer: hongbin ma
Committed: Fri May 5 20:56:52 2017 +0800
--
.../java/org/apache/kylin/metadata/model/ParameterDesc.java| 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/03969236/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 0cf31b1..930dc02 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -96,7 +96,7 @@ public class ParameterDesc implements Serializable {
void setValue(String value) {
this.value = value;
}
-
+
public TblColRef getColRef() {
return colRef;
}
@@ -112,7 +112,7 @@ public class ParameterDesc implements Serializable {
while (p != null) {
if (p.isColumnType())
all.add(p.getColRef());
-
+
p = p.nextParameter;
}
allColRefsIncludingNexts = all;
@@ -145,7 +145,7 @@ public class ParameterDesc implements Serializable {
if (p.isColumnType()) {
if (q.isColumnType() == false)
return false;
-if (this.getColRef().equals(that.getColRef()) == false)
+if (q.getColRef().equals(p.getColRef()) == false)
return false;
} else {
if (q.isColumnType() == true)
[13/50] [abbrv] kylin git commit: KYLIN-2574 RawQueryLastHacker should group by all possible dimensions
KYLIN-2574 RawQueryLastHacker should group by all possible dimensions
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d6fdda2c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d6fdda2c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d6fdda2c
Branch: refs/heads/KYLIN-2624
Commit: d6fdda2c24ac89ec72ff9663a36993f42a70960b
Parents: 1ba9af7
Author: Hongbin Ma
Authored: Fri Apr 28 15:03:43 2017 +0800
Committer: liyang-gmt8
Committed: Mon May 8 16:01:48 2017 +0800
--
.../main/java/org/apache/kylin/cube/RawQueryLastHacker.java| 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6fdda2c/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index 682e48c..d2e3a83 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -46,8 +46,8 @@ public class RawQueryLastHacker {
boolean isSelectAll = sqlDigest.allColumns.isEmpty() ||
sqlDigest.allColumns.equals(sqlDigest.filterColumns);
for (TblColRef col : cubeDesc.listAllColumns()) {
if (cubeDesc.listDimensionColumnsIncludingDerived().contains(col)
|| isSelectAll) {
-if (tupleInfo.hasColumn(col))
-sqlDigest.allColumns.add(col);
+//if (tupleInfo.hasColumn(col))
+sqlDigest.allColumns.add(col);
}
}
@@ -77,4 +77,4 @@ public class RawQueryLastHacker {
}
}
}
-}
\ No newline at end of file
+}
[44/50] [abbrv] kylin git commit: KYLIN-2526 refine find-hive-dependency.sh to support Ubuntu
KYLIN-2526 refine find-hive-dependency.sh to support Ubuntu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0061900b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0061900b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0061900b
Branch: refs/heads/KYLIN-2624
Commit: 0061900bb5bd34c8f0cc1c6e955d34bf34ca48e2
Parents: 5f52cea
Author: Cheng Wang
Authored: Thu May 18 11:04:47 2017 +0800
Committer: liyang-gmt8
Committed: Thu May 18 15:27:36 2017 +0800
--
build/bin/find-hive-dependency.sh | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0061900b/build/bin/find-hive-dependency.sh
--
diff --git a/build/bin/find-hive-dependency.sh
b/build/bin/find-hive-dependency.sh
index 02a47ef..c4c7523 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -100,10 +100,10 @@ fi
function checkFileExist()
{
-files=$1
+files=(`echo $1 | cut -d ":" -f 1- | sed 's/:/ /g'`)
misFiles=0
outputMissFiles=
-for file in ${files//:/ }
+for file in ${files}
do
let allFiles++
if [ ! -f "${file}" ]; then
@@ -111,8 +111,10 @@ function checkFileExist()
let misFiles++
fi
done
-ratio=`echo "scale=3; ${misFiles}/${allFiles}" | bc`
-[[ `echo "$ratio < 0.01" | bc ` -eq 1 ]] || quit "A couple of hive jars
can't be found: ${outputMisFiles}!"
+if [ 0 != ${misFiles} ]; then
+times=`expr ${allFiles} / ${misFiles}`
+[[ ${times} -gt 10 ]] || quit "A couple of hive jars can't be found:
${outputMisFiles}, please export HIVE_LIB='YOUR_LOCAL_HIVE_LIB'"
+fi
}
function validateDirectory()
[41/50] [abbrv] kylin git commit: KYLIN-2627 add simple rollback on ResourceStore
KYLIN-2627 add simple rollback on ResourceStore
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea5cabac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea5cabac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea5cabac
Branch: refs/heads/KYLIN-2624
Commit: ea5cabac95b3d27aa60d6b16263904142e3d2daa
Parents: c9dc7cc
Author: Li Yang
Authored: Wed May 17 17:46:08 2017 +0800
Committer: Roger Shi
Committed: Wed May 17 19:40:34 2017 +0800
--
.../kylin/common/persistence/ResourceStore.java | 103 ++-
.../persistence/LocalFileResourceStoreTest.java | 44
2 files changed, 145 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea5cabac/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index d5fbc2e..0565c66 100644
---
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -18,7 +18,9 @@
package org.apache.kylin.common.persistence;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
@@ -27,6 +29,7 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.NavigableSet;
import java.util.UUID;
@@ -237,6 +240,11 @@ abstract public class ResourceStore {
final public void putResource(String resPath, InputStream content, long
ts) throws IOException {
resPath = norm(resPath);
logger.trace("Directly saving resource " + resPath + " (Store " +
kylinConfig.getMetadataUrl() + ")");
+putResourceCheckpoint(resPath, content, ts);
+}
+
+private void putResourceCheckpoint(String resPath, InputStream content,
long ts) throws IOException {
+beforeChange(resPath);
putResourceImpl(resPath, content, ts);
}
@@ -266,7 +274,7 @@ abstract public class ResourceStore {
dout.close();
buf.close();
-newTS = checkAndPutResourceImpl(resPath, buf.toByteArray(), oldTS,
newTS);
+newTS = checkAndPutResourceCheckpoint(resPath, buf.toByteArray(),
oldTS, newTS);
obj.setLastModified(newTS); // update again the confirmed TS
return newTS;
} catch (IOException e) {
@@ -278,6 +286,11 @@ abstract public class ResourceStore {
}
}
+private long checkAndPutResourceCheckpoint(String resPath, byte[] content,
long oldTS, long newTS) throws IOException {
+beforeChange(resPath);
+return checkAndPutResourceImpl(resPath, content, oldTS, newTS);
+}
+
/**
* checks old timestamp when overwriting existing
*/
@@ -288,7 +301,12 @@ abstract public class ResourceStore {
*/
final public void deleteResource(String resPath) throws IOException {
logger.trace("Deleting resource " + resPath + " (Store " +
kylinConfig.getMetadataUrl() + ")");
-deleteResourceImpl(norm(resPath));
+deleteResourceCheckpoint(norm(resPath));
+}
+
+private void deleteResourceCheckpoint(String resPath) throws IOException {
+beforeChange(resPath);
+deleteResourceImpl(resPath);
}
abstract protected void deleteResourceImpl(String resPath) throws
IOException;
@@ -315,6 +333,87 @@ abstract public class ResourceStore {
//
+ThreadLocal checkpointing = new ThreadLocal<>();
+
+public Checkpoint checkpoint() {
+Checkpoint cp = checkpointing.get();
+if (cp != null)
+throw new IllegalStateException("A checkpoint has been open for
this thread: " + cp);
+
+cp = new Checkpoint();
+checkpointing.set(cp);
+return cp;
+}
+
+private void beforeChange(String resPath) throws IOException {
+Checkpoint cp = checkpointing.get();
+if (cp != null)
+cp.beforeChange(resPath);
+}
+
+public class Checkpoint implements Closeable {
+
+LinkedHashMap origResData = new LinkedHashMap<>();
+LinkedHashMap origResTimestamp = new LinkedHashMap<>();
+
+private void beforeChange(String resPath) throws IOException {
+if (origResData.containsKey(resPath))
+return;
+
+
[24/50] [abbrv] kylin git commit: KYLIN-2609 Grant role access to the project failed on the project manage page.
KYLIN-2609 Grant role access to the project failed on the project manage page.
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6428efa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6428efa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6428efa
Branch: refs/heads/KYLIN-2624
Commit: c6428efae6d0c5c53823666e1bc72ea0bdd9f4e7
Parents: d449c40
Author: 10069681
Authored: Thu May 11 20:19:14 2017 +0800
Committer: Billy Liu
Committed: Thu May 11 21:51:30 2017 +0800
--
webapp/app/js/controllers/access.js | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6428efa/webapp/app/js/controllers/access.js
--
diff --git a/webapp/app/js/controllers/access.js
b/webapp/app/js/controllers/access.js
index 0e0664a..6187f9a 100644
--- a/webapp/app/js/controllers/access.js
+++ b/webapp/app/js/controllers/access.js
@@ -55,6 +55,7 @@ KylinApp.controller('AccessCtrl', function ($scope,
AccessService, MessageServic
//MessageService.sendMsg('Access granted!', 'success', {});
SweetAlert.swal('Success!', 'Access granted!', 'success');
}, function (e) {
+ grantRequst.uuid = uuid;
if (e.status == 404) {
//MessageService.sendMsg('User not found!', 'error', {});
SweetAlert.swal('Oops...', 'User not found!!', 'error');
[42/50] [abbrv] kylin git commit: KYLIN-2635 fix IT
KYLIN-2635 fix IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f52cea4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f52cea4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f52cea4
Branch: refs/heads/KYLIN-2624
Commit: 5f52cea4f7ffbb0fa885df650803156cd1daa5d2
Parents: ea5caba
Author: Hongbin Ma
Authored: Wed May 17 21:26:09 2017 +0800
Committer: Roger Shi
Committed: Thu May 18 11:02:49 2017 +0800
--
.../org/apache/kylin/metadata/filter/CompareTupleFilter.java | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f52cea4/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 2771250..c091c7c 100755
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -167,7 +167,13 @@ public class CompareTupleFilter extends TupleFilter {
return true;
else
return false;
+} else {
+if (operator == FilterOperatorEnum.ISNOTNULL)
+return true;
+else if (operator == FilterOperatorEnum.ISNULL)
+return false;
}
+
if (cs.isNull(firstCondValue)) {
return false;
}
[37/50] [abbrv] kylin git commit: Revert "KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in Broadcaster"
Revert "KYLIN-2619 Use newCachedThreadPool instead of newFixedThreadPool in
Broadcaster"
This reverts commit 1c3ed36f7dcf2152b17966bb69dd3211ce4e8aba.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/858d9475
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/858d9475
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/858d9475
Branch: refs/heads/KYLIN-2624
Commit: 858d947518b308c1a33b832ea8ef49144285c02e
Parents: 1c3ed36
Author: kangkaisen
Authored: Tue May 16 09:48:22 2017 +0800
Committer: kangkaisen
Committed: Tue May 16 09:48:22 2017 +0800
--
.../java/org/apache/kylin/common/restclient/RestClient.java | 4
.../java/org/apache/kylin/metadata/cachesync/Broadcaster.java | 7 ++-
2 files changed, 2 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/858d9475/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 470386b..33a4e7a 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -287,8 +287,4 @@ public class RestClient {
return result.toString();
}
-@Override
-public String toString() {
-return String.format("RestClient(%s:%d)", host, port);
-}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/858d9475/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index e787dfd..1394f7b 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -35,7 +35,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.DaemonThreadFactory;
-import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +116,7 @@ public class Broadcaster {
for (String node : config.getRestServers()) {
restClients.add(new RestClient(node));
}
-final ExecutorService wipingCachePool =
Executors.newCachedThreadPool(new DaemonThreadFactory());
+final ExecutorService wipingCachePool =
Executors.newFixedThreadPool(restClients.size(), new DaemonThreadFactory());
while (true) {
try {
final BroadcastEvent broadcastEvent =
broadcastEvents.takeFirst();
@@ -126,10 +125,8 @@ public class Broadcaster {
wipingCachePool.execute(new Runnable() {
@Override
public void run() {
-try (SetThreadName ignored = new
SetThreadName("CacheWiper %s %s", restClient, broadcastEvent)) {
-logger.info("{} wipe cache {}",
restClient, broadcastEvent);
+try {
restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(),
broadcastEvent.getCacheKey());
-logger.info("{} wipe cache {}
success", restClient, broadcastEvent);
} catch (IOException e) {
logger.warn("Thread failed during wipe
cache at " + broadcastEvent, e);
}
[06/50] [abbrv] kylin git commit: minor, check random port availability before using it
minor, check random port availability before using it
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d738544e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d738544e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d738544e
Branch: refs/heads/KYLIN-2624
Commit: d738544eeb03235ed2cf9e2f4509c1d09d98db8a
Parents: 7df407d
Author: Yang Li
Authored: Sun May 7 09:33:45 2017 +0800
Committer: Yang Li
Committed: Sun May 7 14:52:14 2017 +0800
--
.../org/apache/kylin/common/util/CheckUtil.java | 50 +++-
.../kylin/rest/service/CacheServiceTest.java| 5 +-
2 files changed, 51 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d738544e/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
index ae189f7..0f75ff2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -17,10 +17,16 @@
*/
package org.apache.kylin.common.util;
+import java.io.IOException;
+import java.net.DatagramSocket;
+import java.net.ServerSocket;
+import java.util.Random;
+
+import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CheckUtil {
-public static final org.slf4j.Logger logger =
LoggerFactory.getLogger(CheckUtil.class);
+public static final Logger logger =
LoggerFactory.getLogger(CheckUtil.class);
public static boolean checkCondition(boolean condition, String message,
Object... args) {
if (condition) {
@@ -30,4 +36,46 @@ public class CheckUtil {
return false;
}
}
+
+public static int randomAvailablePort(int minPort, int maxPort) {
+Random rand = new Random();
+for (int i = 0; i < 100; i++) {
+int p = minPort + rand.nextInt(maxPort - minPort);
+if (checkPortAvailable(p))
+return p;
+}
+throw new RuntimeException("Failed to get random available port
between [" + minPort + "," + maxPort + ")");
+}
+
+/**
+ * Checks to see if a specific port is available.
+ *
+ * @param port the port to check for availability
+ */
+public static boolean checkPortAvailable(int port) {
+ServerSocket ss = null;
+DatagramSocket ds = null;
+try {
+ss = new ServerSocket(port);
+ss.setReuseAddress(true);
+ds = new DatagramSocket(port);
+ds.setReuseAddress(true);
+return true;
+} catch (IOException e) {
+} finally {
+if (ds != null) {
+ds.close();
+}
+
+if (ss != null) {
+try {
+ss.close();
+} catch (IOException e) {
+/* should not be thrown */
+}
+}
+}
+
+return false;
+}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d738544e/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
--
diff --git
a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 3601ec1..6410aaa 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -24,12 +24,12 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.CheckUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
@@ -74,8 +74,7 @@ public class CacheServiceTest extends
LocalFileMetadataTestCase {
public static void beforeClass() throws Exception {
staticCreateTestMetadata();
-Random random = new Random();
-int port = random.nextInt(1) + 4;
+int port = CheckUtil.randomAvailablePort(4, 5);
logger.info("Chosen port for CacheServiceTest is " + port);
configA = KylinConfig.getInstanceFromEnv();
configA.setProperty("kylin.server.cluster-servers", "localhost:" +
port);
[18/50] [abbrv] kylin git commit: KYLIN-2599 select * in subquery fail due to bug in hackSelectStar
KYLIN-2599 select * in subquery fail due to bug in hackSelectStar
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3d81832
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3d81832
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3d81832
Branch: refs/heads/KYLIN-2624
Commit: c3d81832f796ab281526d3ac1cd97b11eb986a92
Parents: 9dc6553
Author: Hongbin Ma
Authored: Tue May 9 21:42:47 2017 +0800
Committer: liyang-gmt8
Committed: Wed May 10 09:41:40 2017 +0800
--
.../calcite/sql2rel/SqlToRelConverter.java | 11 +
.../src/test/resources/query/sql/query105.sql | 26
2 files changed, 37 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3d81832/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index c199c31..a520aa6 100644
---
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -47,6 +47,7 @@ import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.core.Sample;
import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.core.Uncollect;
import org.apache.calcite.rel.logical.LogicalAggregate;
import org.apache.calcite.rel.logical.LogicalCorrelate;
@@ -611,6 +612,16 @@ public class SqlToRelConverter {
} else {
return root;
}
+
+RelNode tableScanOrJoin = null;
+RelNode rootProjInput = rootPrj.getInput(0) instanceof LogicalFilter ?
rootPrj.getInput(0).getInput(0): rootPrj.getInput(0);
+if(rootProjInput instanceof TableScan || rootProjInput instanceof
LogicalJoin) {
+tableScanOrJoin = rootProjInput;
+}
+if(tableScanOrJoin == null) {
+return root;
+}
+
//
RelNode input = rootPrj.getInput();
//if (!(//
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3d81832/kylin-it/src/test/resources/query/sql/query105.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query105.sql
b/kylin-it/src/test/resources/query/sql/query105.sql
new file mode 100644
index 000..a467b65
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query105.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select fact.lstg_format_name from
+
+ (select * from test_kylin_fact where cal_dt > date'2010-01-01' ) as fact
+
+ group by fact.lstg_format_name
+
+ order by CASE WHEN fact.lstg_format_name IS NULL THEN 'sdf' ELSE
fact.lstg_format_name END
+
[29/50] [abbrv] kylin git commit: KYLIN-2598 increase default kylin.query.translated-in-clause-max-size
KYLIN-2598 increase default kylin.query.translated-in-clause-max-size
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c0038db
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c0038db
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c0038db
Branch: refs/heads/KYLIN-2624
Commit: 7c0038db4aab12c6fbd8f249b104a473e1d5a61c
Parents: b08d278
Author: Hongbin Ma
Authored: Fri May 12 13:11:01 2017 +0800
Committer: Dong Li
Committed: Fri May 12 13:12:08 2017 +0800
--
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c0038db/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 10209d7..b620924 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -845,7 +845,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public int getTranslatedInClauseMaxSize() {
-return
Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size",
"1024"));
+return
Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size",
String.valueOf(1024 * 1024)));
}
public int getLargeQueryThreshold() {
[20/50] [abbrv] kylin git commit: The return type of tinyint for sum measure should be bigint
The return type of tinyint for sum measure should be bigint
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1264ff39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1264ff39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1264ff39
Branch: refs/heads/KYLIN-2624
Commit: 1264ff39b740f9515653d30e0712739bc717d438
Parents: 901b10c
Author: kangkaisen
Authored: Wed May 10 20:03:40 2017 +0800
Committer: kangkaisen
Committed: Wed May 10 20:03:40 2017 +0800
--
webapp/app/js/controllers/cubeMeasures.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/1264ff39/webapp/app/js/controllers/cubeMeasures.js
--
diff --git a/webapp/app/js/controllers/cubeMeasures.js
b/webapp/app/js/controllers/cubeMeasures.js
index 73a6cd0..824e2ca 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -367,7 +367,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope,
$modal,MetaModel,cubes
switch($scope.newMeasure.function.expression){
case "SUM":
-
if(colType==="smallint"||colType==="int"||colType==="bigint"||colType==="integer"){
+
if(colType==="tinyint"||colType==="smallint"||colType==="int"||colType==="bigint"||colType==="integer"){
$scope.newMeasure.function.returntype= 'bigint';
}else{
if(colType.indexOf('decimal')!=-1||colType==="double"||colType==="float"){
[17/50] [abbrv] kylin git commit: KYLIN-2597 Deal with trivial expression in filters like x = 1+2
KYLIN-2597 Deal with trivial expression in filters like x = 1+2
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/529b1be2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/529b1be2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/529b1be2
Branch: refs/heads/KYLIN-2624
Commit: 529b1be2f9f0f1c7bfb2798928b8c6c60cb581ea
Parents: 9a812d5
Author: Hongbin Ma
Authored: Tue May 9 17:30:03 2017 +0800
Committer: Dong Li
Committed: Tue May 9 20:30:21 2017 +0800
--
.../test/resources/query/sql_limit/query03.sql | 31 +++
.../kylin/query/relnode/OLAPFilterRel.java | 55
2 files changed, 86 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/529b1be2/kylin-it/src/test/resources/query/sql_limit/query03.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_limit/query03.sql
b/kylin-it/src/test/resources/query/sql_limit/query03.sql
new file mode 100644
index 000..4844395
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_limit/query03.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ test_kylin_fact.lstg_format_name ,test_kylin_fact.cal_dt ,
sum(test_kylin_fact.price) as GMV, count(*) as TRANS_CNT
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ where test_kylin_fact.seller_id = 1002 + 4
+
+ group by seller_id,test_kylin_fact.cal_dt,lstg_format_name
+ limit 10
http://git-wip-us.apache.org/repos/asf/kylin/blob/529b1be2/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 0ee9204..06ad0c6 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.relnode;
+import java.math.BigDecimal;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.LinkedList;
@@ -68,6 +69,7 @@ import org.apache.kylin.metadata.filter.function.Functions;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -140,6 +142,16 @@ public class OLAPFilterRel extends Filter implements
OLAPRel {
case OTHER_FUNCTION:
filter = Functions.getFunctionTupleFilter(op.getName());
break;
+case PLUS:
+case MINUS:
+case TIMES:
+case DIVIDE:
+TupleFilter f = dealWithTrivialExpr(call);
+if (f != null) {
+// is a trivial expr
+return f;
+}
+//else go to default
default:
filter = new
UnsupportedTupleFilter(FilterOperatorEnum.UNSUPPORTED);
break;
@@ -166,6 +178,49 @@ public class OLAPFilterRel extends Filter implements
OLAPRel {
return filter;
}
+//KYLIN-2597 - Deal with trivial expression in filters like x = 1 + 2
+private TupleFilter dealWithTrivialExpr(RexCall call) {
+ImmutableList operators = call.operands;
+if (operators.size() != 2) {
+return null;
+}
+
+BigDecimal left = null;
+BigDecimal right = null;
+
[03/50] [abbrv] kylin git commit: Minor, remove useless imports.
Minor, remove useless imports. Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a0598e3 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a0598e3 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a0598e3 Branch: refs/heads/KYLIN-2624 Commit: 9a0598e31592b204f92e9f30843103b5552a9883 Parents: 5e6ed11 Author: nichunen Authored: Fri May 5 10:22:40 2017 +0800 Committer: hongbin ma Committed: Fri May 5 10:42:44 2017 +0800 -- .../main/java/org/apache/kylin/rest/controller/JobController.java | 2 -- 1 file changed, 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/9a0598e3/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java -- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java index 7b1de37..2e6f6cd 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java @@ -28,9 +28,7 @@ import java.util.Map; import org.apache.kylin.job.JobInstance; import org.apache.kylin.job.constant.JobStatusEnum; import org.apache.kylin.job.constant.JobTimeFilterEnum; -import org.apache.kylin.job.exception.JobException; import org.apache.kylin.rest.exception.InternalErrorException; -import org.apache.kylin.rest.exception.NotFoundException; import org.apache.kylin.rest.request.JobListRequest; import org.apache.kylin.rest.service.JobService; import org.slf4j.Logger;
[09/50] [abbrv] kylin git commit: KYLIN-2428 more cleaning
KYLIN-2428 more cleaning
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8b277d5e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8b277d5e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8b277d5e
Branch: refs/heads/KYLIN-2624
Commit: 8b277d5e492d8d54e6fd1f38eef7cf8d174f2b00
Parents: 0396923
Author: Hongbin Ma
Authored: Fri May 5 21:12:46 2017 +0800
Committer: liyang-gmt8
Committed: Sun May 7 21:22:28 2017 +0800
--
assembly/pom.xml| 4 --
core-common/pom.xml | 5 --
core-cube/pom.xml | 4 +-
core-dictionary/pom.xml | 5 --
engine-mr/pom.xml | 5 --
engine-spark/pom.xml| 8 +--
kylin-it/pom.xml| 5 --
pom.xml | 60 ---
server/pom.xml | 15 -
source-hive/pom.xml | 5 --
source-kafka/pom.xml| 7 +--
.../kylin/source/kafka/DateTimeParser.java | 62
storage-hbase/pom.xml | 4 +-
tool-assembly/pom.xml | 5 ++
14 files changed, 51 insertions(+), 143 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 121ccee..dae7152 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -207,10 +207,6 @@
${shadeBase}.org.apache.commons.codec
-
org.apache.commons.collections
-
${shadeBase}.org.apache.commons.collections
-
-
org.apache.commons.lang3
${shadeBase}.org.apache.commons.lang3
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 55f025f..8852743 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -64,11 +64,6 @@
-org.apache.hadoop
-hadoop-common
-provided
-
-
junit
junit
test
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index cd382ef..930b6cd 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -44,12 +44,12 @@
kylin-core-dictionary
+
+
commons-collections
commons-collections
-
-
commons-lang
commons-lang
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 09dde1e..c292982 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -41,11 +41,6 @@
-org.apache.hadoop
-hadoop-common
-provided
-
-
org.apache.kylin
kylin-core-common
test-jar
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/engine-mr/pom.xml
--
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 8aab79d..f2cedf6 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -55,11 +55,6 @@
org.apache.hadoop
-hadoop-common
-provided
-
-
-org.apache.hadoop
hadoop-hdfs
provided
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/engine-spark/pom.xml
--
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index c5d23ce..93b6f9b 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -69,12 +69,6 @@
-org.apache.hadoop
-hadoop-common
-provided
-
-
-
org.apache.hbase
hbase-common
provided
@@ -120,4 +114,4 @@
-
\ No newline at end of fil
[22/50] [abbrv] kylin git commit: KYLIN-2605 Execute sample.sh script failure in hdfs federation environment
KYLIN-2605 Execute sample.sh script failure in hdfs federation environment
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d449c40d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d449c40d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d449c40d
Branch: refs/heads/KYLIN-2624
Commit: d449c40d58b8af10ebc638be4aadb80e09787a9d
Parents: b3310dc
Author: 10069681
Authored: Thu May 11 11:19:01 2017 +0800
Committer: Billy Liu
Committed: Thu May 11 13:34:24 2017 +0800
--
build/bin/sample.sh | 25 --
examples/sample_cube/create_sample_tables.sql | 10 -
2 files changed, 23 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d449c40d/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 617b671..10900bb 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -31,9 +31,20 @@ else
hadoop_conf_param="--config ${kylin_hadoop_conf_dir}"
fi
-echo "Loading sample data into HDFS tmp path: /tmp/kylin/sample_cube/data"
-hadoop ${hadoop_conf_param} fs -mkdir -p /tmp/kylin/sample_cube/data
-hadoop ${hadoop_conf_param} fs -put * /tmp/kylin/sample_cube/data/
+if [ -z "$1" ]; then
+hdfs_tmp_dir=/tmp/kylin
+else
+hdfs_tmp_dir=$1
+fi
+
+echo "Loading sample data into HDFS tmp path: ${hdfs_tmp_dir}/sample_cube/data"
+hadoop ${hadoop_conf_param} fs -mkdir -p ${hdfs_tmp_dir}/sample_cube/data
+if [ $? != 0 ]
+then
+quit "Failed to create ${hdfs_tmp_dir}/sample_cube/data. Please make sure
the user has right to access ${hdfs_tmp_dir}/sample_cube/data or usage:
sample.sh hdfs_tmp_dir"
+fi
+
+hadoop ${hadoop_conf_param} fs -put * ${hdfs_tmp_dir}/sample_cube/data/
hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh
kylin.source.hive.client`
sample_database=`bash ${KYLIN_HOME}/bin/get-properties.sh
kylin.source.hive.database-for-flat-table`
@@ -51,14 +62,14 @@ then
else
beeline_params=${beeline_params/${hive2_url}/${hive2_url}${sample_database}}
fi
-beeline ${beeline_params} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
+beeline ${beeline_params} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
else
hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database
-hive --database $sample_database -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
+hive --database $sample_database --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
fi
echo "Sample hive tables are created successfully; Going to create sample
cube..."
-hadoop ${hadoop_conf_param} fs -rm -r /tmp/kylin/sample_cube
+hadoop ${hadoop_conf_param} fs -rm -r ${hdfs_tmp_dir}/sample_cube
# set engine type and storage type to cube desc
default_engine_type=`bash ${KYLIN_HOME}/bin/get-properties.sh
kylin.engine.default`
@@ -85,4 +96,4 @@ ls -1 DEFAULT.KYLIN_*.json|sed "s/\(DEFAULT\)\(.*\)\.json/mv
& $sample_database\
cd ${KYLIN_HOME}
hbase org.apache.hadoop.util.RunJar ${job_jar}
org.apache.kylin.common.persistence.ResourceTool upload
${KYLIN_HOME}/sample_cube/metadata || { exit 1; }
echo "Sample cube is created successfully in project 'learn_kylin'."
-echo "Restart Kylin server or reload the metadata from web UI to see the
change."
+echo "Restart Kylin server or reload the metadata from web UI to see the
change."
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d449c40d/examples/sample_cube/create_sample_tables.sql
--
diff --git a/examples/sample_cube/create_sample_tables.sql
b/examples/sample_cube/create_sample_tables.sql
index 24f26f8..c33a99e 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -216,8 +216,8 @@ COMMENT 'Sales order table, fact table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv'
OVERWRITE INTO TABLE KYLIN_SALES;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv'
OVERWRITE INTO TABLE KYLIN_ACCOUNT;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv'
OVERWRITE INTO TABLE KYLIN_COUNTRY;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv'
OVERWRITE INTO TABLE KYLIN_CAL_DT;
-LOAD DATA INPATH
'/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE
INTO TABLE KYLIN_CATEGORY_GROUPINGS;
+LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_
[28/50] [abbrv] kylin git commit: KYLIN-2610 Optimize BuiltInFunctionTransformer performance
KYLIN-2610 Optimize BuiltInFunctionTransformer performance
KYLIN-2610 fix three percent sign bug
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3ba338e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3ba338e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3ba338e
Branch: refs/heads/KYLIN-2624
Commit: b3ba338eaf889d42e02870007c1876eb32c884ee
Parents: 3976420
Author: Hongbin Ma
Authored: Thu May 11 20:26:14 2017 +0800
Committer: Hongbin Ma
Committed: Fri May 12 13:00:43 2017 +0800
--
.../kylin/dict/BuiltInFunctionTransformer.java | 9 +-
.../metadata/filter/function/BuiltInMethod.java | 39 +++--
.../metadata/filter/function/LikeMatchers.java | 141 +++
3 files changed, 177 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b3ba338e/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 6ef2f96..5196593 100755
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -57,13 +57,13 @@ public class BuiltInFunctionTransformer implements
ITupleFilterTransformer {
//normal case
translated = translateCompareTupleFilter((CompareTupleFilter)
tupleFilter);
if (translated != null) {
-logger.debug("Translated {{}} to IN clause: {{}}",
tupleFilter, translated);
+logger.debug("Translated {{}} to IN clause. ", tupleFilter);
}
} else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
-//like case
+//like,tolower case
translated =
translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
if (translated != null) {
-logger.debug("Translated {{}} to IN clause: {{}}",
tupleFilter, translated);
+logger.debug("Translated {{}} to IN clause. ", tupleFilter);
}
} else if (tupleFilter instanceof LogicalTupleFilter) {
@SuppressWarnings("unchecked")
@@ -99,6 +99,7 @@ public class BuiltInFunctionTransformer implements
ITupleFilterTransformer {
translated.addChild(new ConstantTupleFilter(dictVal));
}
}
+logger.debug("getting a in clause with {} children",
translated.getChildren().size());
} catch (Exception e) {
logger.debug(e.getMessage());
return null;
@@ -175,4 +176,4 @@ public class BuiltInFunctionTransformer implements
ITupleFilterTransformer {
}
return translated;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b3ba338e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
index 2f28fae..31ee297 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
@@ -18,8 +18,11 @@
package org.apache.kylin.metadata.filter.function;
+import static
org.apache.kylin.metadata.filter.function.LikeMatchers.LikeMatcher;
+
import java.lang.reflect.Method;
-import java.util.regex.Pattern;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.lang3.reflect.MethodUtils;
@@ -28,9 +31,15 @@ import com.google.common.collect.ImmutableMap;
public enum BuiltInMethod {
UPPER(BuiltInMethod.class, "upper", String.class),
LOWER(BuiltInMethod.class, "lower", String.class),
SUBSTRING(BuiltInMethod.class, "substring", String.class, int.class,
int.class), CHAR_LENGTH(BuiltInMethod.class, "charLength", String.class),
LIKE(BuiltInMethod.class, "like", String.class, String.class),
INITCAP(BuiltInMethod.class, "initcap", String.class);
public final Method method;
-
public static final ImmutableMap MAP;
+private static ThreadLocal> likePatterns = new
ThreadLocal>() {
+@Override
+public Map initialValue() {
+return new HashMap<>();
+}
+};
+
static {
final ImmutableMap.Builder builder =
ImmutableMap.builder();
for (BuiltInMethod
[34/50] [abbrv] kylin git commit: KYLIN-2557-The-kylin-will-start-failed
KYLIN-2557-The-kylin-will-start-failed
Signed-off-by: Yang Li
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6a8a00b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6a8a00b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6a8a00b
Branch: refs/heads/KYLIN-2624
Commit: e6a8a00b89dbdf922405f8586b43d1c9b99cff90
Parents: c8524ef
Author: 10069681
Authored: Tue May 9 10:47:50 2017 +0800
Committer: Yang Li
Committed: Sun May 14 19:45:48 2017 +0800
--
.../kylin/storage/hbase/HBaseConnection.java| 21 +++-
1 file changed, 20 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6a8a00b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index f0b9428..ef82641 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.lock.DistributedLock;
import org.apache.kylin.common.persistence.StorageException;
import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
@@ -66,6 +67,8 @@ public class HBaseConnection {
private static ExecutorService coprocessorPool = null;
+private static DistributedLock lock = null;
+
static {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
@@ -267,6 +270,7 @@ public class HBaseConnection {
public static void createHTableIfNeeded(Connection conn, String table,
String... families) throws IOException {
Admin hbase = conn.getAdmin();
TableName tableName = TableName.valueOf(table);
+boolean hasLock = false;
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
@@ -290,6 +294,14 @@ public class HBaseConnection {
return;
}
+lock =
KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentProcess();
+hasLock = lock.lock(getLockPath(table), Long.MAX_VALUE);
+
+if (tableExists(conn, table)) {
+logger.debug("HTable '" + table + "' already exists");
+return;
+}
+
logger.debug("Creating HTable '" + table + "'");
HTableDescriptor desc = new
HTableDescriptor(TableName.valueOf(table));
@@ -307,6 +319,9 @@ public class HBaseConnection {
logger.debug("HTable '" + table + "' created");
} finally {
hbase.close();
+if (hasLock && lock != null) {
+lock.unlock(getLockPath(table));
+}
}
}
@@ -350,4 +365,8 @@ public class HBaseConnection {
}
}
-}
+private static String getLockPath(String pathName) {
+return "/create_htable/" + pathName + "/lock";
+}
+
+}
\ No newline at end of file
[32/50] [abbrv] kylin git commit: KYLIN-2581 fix possible deadlock in Broadcaster
KYLIN-2581 fix possible deadlock in Broadcaster
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fd671d16
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fd671d16
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fd671d16
Branch: refs/heads/KYLIN-2624
Commit: fd671d165aa59c734b2f273ffae6d1017dc61153
Parents: 320e56a
Author: Yang Li
Authored: Sun May 14 13:34:58 2017 +0800
Committer: Yang Li
Committed: Sun May 14 13:35:29 2017 +0800
--
.../kylin/metadata/cachesync/Broadcaster.java | 66 ++--
1 file changed, 32 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd671d16/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 5b45d9e..1394f7b 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -142,7 +142,7 @@ public class Broadcaster {
}
public void registerListener(Listener listener, String... entities) {
-synchronized (CACHE) {
+synchronized (listenerMap) {
// ignore re-registration
List all = listenerMap.get(SYNC_ALL);
if (all != null && all.contains(listener)) {
@@ -181,43 +181,41 @@ public class Broadcaster {
}
public void notifyListener(String entity, Event event, String cacheKey)
throws IOException {
-synchronized (CACHE) {
-List list = listenerMap.get(entity);
-if (list == null)
-return;
+List list = listenerMap.get(entity);
+if (list == null)
+return;
-logger.debug("Broadcasting metadata change: entity=" + entity + ",
event=" + event + ", cacheKey=" + cacheKey + ", listeners=" + list);
+logger.debug("Broadcasting metadata change: entity=" + entity + ",
event=" + event + ", cacheKey=" + cacheKey + ", listeners=" + list);
-// prevents concurrent modification exception
-list = Lists.newArrayList(list);
-switch (entity) {
-case SYNC_ALL:
-for (Listener l : list) {
-l.onClearAll(this);
-}
-clearCache(); // clear broadcaster too in the end
-break;
-case SYNC_PRJ_SCHEMA:
-ProjectManager.getInstance(config).clearL2Cache();
-for (Listener l : list) {
-l.onProjectSchemaChange(this, cacheKey);
-}
-break;
-case SYNC_PRJ_DATA:
-ProjectManager.getInstance(config).clearL2Cache(); // cube's
first becoming ready leads to schema change too
-for (Listener l : list) {
-l.onProjectDataChange(this, cacheKey);
-}
-break;
-default:
-for (Listener l : list) {
-l.onEntityChange(this, entity, event, cacheKey);
-}
-break;
+// prevents concurrent modification exception
+list = Lists.newArrayList(list);
+switch (entity) {
+case SYNC_ALL:
+for (Listener l : list) {
+l.onClearAll(this);
}
-
-logger.debug("Done broadcasting metadata change: entity=" + entity
+ ", event=" + event + ", cacheKey=" + cacheKey);
+clearCache(); // clear broadcaster too in the end
+break;
+case SYNC_PRJ_SCHEMA:
+ProjectManager.getInstance(config).clearL2Cache();
+for (Listener l : list) {
+l.onProjectSchemaChange(this, cacheKey);
+}
+break;
+case SYNC_PRJ_DATA:
+ProjectManager.getInstance(config).clearL2Cache(); // cube's first
becoming ready leads to schema change too
+for (Listener l : list) {
+l.onProjectDataChange(this, cacheKey);
+}
+break;
+default:
+for (Listener l : list) {
+l.onEntityChange(this, entity, event, cacheKey);
+}
+break;
}
+
+logger.debug("Done broadcasting metadata change: entity=" + entity +
", event=" + event + ", cacheKey=" + cacheKey);
}
/**
[31/50] [abbrv] kylin git commit: KYLIN-2607 Add http timeout for RestClient
KYLIN-2607 Add http timeout for RestClient
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/320e56a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/320e56a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/320e56a5
Branch: refs/heads/KYLIN-2624
Commit: 320e56a532cf6d4968430c8111d239961a07c7f8
Parents: b3ba338
Author: kangkaisen
Authored: Thu May 11 14:57:29 2017 +0800
Committer: kangkaisen
Committed: Fri May 12 16:47:39 2017 +0800
--
.../apache/kylin/common/restclient/RestClient.java | 16 +---
1 file changed, 13 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/320e56a5/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 825c67f..33a4e7a 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -40,6 +40,9 @@ import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.apache.kylin.common.util.JsonUtil;
@@ -59,6 +62,9 @@ public class RestClient {
protected static Pattern fullRestPattern =
Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?");
+private static final int HTTP_CONNECTION_TIMEOUT_MS = 3;
+private static final int HTTP_SOCKET_TIMEOUT_MS = 12;
+
public static boolean matchFullRestPattern(String uri) {
Matcher m = fullRestPattern.matcher(uri);
return m.matches();
@@ -93,7 +99,11 @@ public class RestClient {
this.password = password;
this.baseUrl = "http://"; + host + ":" + port + "/kylin/api";
-client = new DefaultHttpClient();
+final HttpParams httpParams = new BasicHttpParams();
+HttpConnectionParams.setSoTimeout(httpParams, HTTP_SOCKET_TIMEOUT_MS);
+HttpConnectionParams.setConnectionTimeout(httpParams,
HTTP_CONNECTION_TIMEOUT_MS);
+
+client = new DefaultHttpClient(httpParams);
if (userName != null && password != null) {
CredentialsProvider provider = new BasicCredentialsProvider();
@@ -263,14 +273,14 @@ public class RestClient {
InputStreamReader reader = null;
BufferedReader rd = null;
StringBuffer result = new StringBuffer();
-try{
+try {
reader = new InputStreamReader(response.getEntity().getContent());
rd = new BufferedReader(reader);
String line = null;
while ((line = rd.readLine()) != null) {
result.append(line);
}
-}finally {
+} finally {
IOUtils.closeQuietly(reader);
IOUtils.closeQuietly(rd);
}
[14/50] [abbrv] kylin git commit: KYLIN-2582 'Server Config' should be refreshed automatically in web page 'System', after we update it successfully.
KYLIN-2582 'Server Config' should be refreshed automatically in web page
'System', after we update it successfully.
Signed-off-by: Yang Li
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/42f7e7da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/42f7e7da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/42f7e7da
Branch: refs/heads/KYLIN-2624
Commit: 42f7e7dad7f34f9d0849c72f1a20c3e2d4475b99
Parents: 41c7a4d
Author: 10069681
Authored: Wed May 3 11:11:02 2017 +0800
Committer: Yang Li
Committed: Tue May 9 07:06:03 2017 +0800
--
webapp/app/js/controllers/admin.js | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/42f7e7da/webapp/app/js/controllers/admin.js
--
diff --git a/webapp/app/js/controllers/admin.js
b/webapp/app/js/controllers/admin.js
index 783ab17..10e2c38 100644
--- a/webapp/app/js/controllers/admin.js
+++ b/webapp/app/js/controllers/admin.js
@@ -71,6 +71,7 @@ KylinApp.controller('AdminCtrl', function ($scope,
AdminService, CacheService, T
if (isConfirm) {
CacheService.reloadConfig({}, function () {
SweetAlert.swal('Success!', 'config reload successfully', 'success');
+ $scope.getConfig();
}, function (e) {
if (e.data && e.data.exception) {
var message = e.data.exception;
@@ -214,6 +215,7 @@ KylinApp.controller('AdminCtrl', function ($scope,
AdminService, CacheService, T
$modal.open({
templateUrl: 'updateConfig.html',
controller: updateConfigCtrl,
+ scope: $scope,
resolve: {}
});
}
@@ -256,11 +258,10 @@ KylinApp.controller('AdminCtrl', function ($scope,
AdminService, CacheService, T
$modalInstance.dismiss('cancel');
};
$scope.update = function () {
-
-
AdminService.updateConfig({}, {key: $scope.state.key, value:
$scope.state.value}, function (result) {
SweetAlert.swal('Success!', 'Config updated successfully!', 'success');
$modalInstance.dismiss();
+$scope.getConfig();
}, function (e) {
if (e.data && e.data.exception) {
var message = e.data.exception;
[25/50] [abbrv] kylin git commit: KYLIN-2595 Result query should use alias name to replace dimension column name when used Keyword As
KYLIN-2595 Result query should use alias name to replace dimension column name
when used Keyword As
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/39764200
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/39764200
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/39764200
Branch: refs/heads/KYLIN-2624
Commit: 397642009ed6056d7ee5ee66f51f7f67026933d4
Parents: c6428ef
Author: wuyingjun
Authored: Thu May 11 23:17:09 2017 +0800
Committer: Billy Liu
Committed: Thu May 11 23:45:45 2017 +0800
--
webapp/app/js/controllers/query.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/39764200/webapp/app/js/controllers/query.js
--
diff --git a/webapp/app/js/controllers/query.js
b/webapp/app/js/controllers/query.js
index 6be915b..945ddef 100644
--- a/webapp/app/js/controllers/query.js
+++ b/webapp/app/js/controllers/query.js
@@ -185,14 +185,14 @@ KylinApp
angular.forEach(result.results, function (row, index) {
var oneRow = {};
angular.forEach(result.columnMetas, function (meta, metaIndex)
{
-oneRow[meta.name] = row[metaIndex];
+oneRow[meta.label] = row[metaIndex];
});
data.push(oneRow);
});
var columnDefs = [];
angular.forEach(result.columnMetas, function (meta, metaIndex) {
-columnDefs.push({field: meta.name, width: 120});
+columnDefs.push({field: meta.label, width: 120});
});
if (oneQuery.result.results) {
[08/50] [abbrv] kylin git commit: KYLIN-2578 fix ITDistributedSchedulerBaseTest
KYLIN-2578 fix ITDistributedSchedulerBaseTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41c7a4d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41c7a4d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41c7a4d9
Branch: refs/heads/KYLIN-2624
Commit: 41c7a4d9417eb75dad5f0da4e5a07a14e8f7e056
Parents: 263a1e8
Author: Yang Li
Authored: Sun May 7 20:36:24 2017 +0800
Committer: Yang Li
Committed: Sun May 7 20:36:24 2017 +0800
--
.../kylin/job/impl/threadpool/DistributedScheduler.java | 4 ++--
.../apache/kylin/job/BaseTestDistributedScheduler.java | 12 ++--
2 files changed, 8 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/41c7a4d9/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index 0714d90..6d12b8b 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -304,11 +304,11 @@ public class DistributedScheduler implements
Scheduler, Conn
}
}
-public String getLockPath(String pathName) {
+public static String getLockPath(String pathName) {
return ZOOKEEPER_LOCK_PATH + "/" + pathName;
}
-private String getWatchPath() {
+private static String getWatchPath() {
return ZOOKEEPER_LOCK_PATH;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/41c7a4d9/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index 3576d18..48ef923 100644
---
a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++
b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -163,8 +163,8 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
}
}
-boolean lock(ZookeeperDistributedLock jobLock, String cubeName) {
-return jobLock.lock(getLockPath(cubeName));
+boolean lock(ZookeeperDistributedLock jobLock, String segName) {
+return jobLock.lock(DistributedScheduler.getLockPath(segName));
}
private static void initZk() {
@@ -177,8 +177,8 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
zkClient.start();
}
-String getServerName(String cubeName) {
-String lockPath = getLockPath(cubeName);
+String getServerName(String segName) {
+String lockPath = getFullLockPath(segName);
String serverName = null;
if (zkClient.getState().equals(CuratorFrameworkState.STARTED)) {
try {
@@ -193,7 +193,7 @@ public class BaseTestDistributedScheduler extends
HBaseMetadataTestCase {
return serverName;
}
-private String getLockPath(String pathName) {
-return DistributedScheduler.ZOOKEEPER_LOCK_PATH + "/" +
kylinConfig1.getMetadataUrlPrefix() + "/" + pathName;
+private String getFullLockPath(String segName) {
+return "/kylin/" + kylinConfig1.getMetadataUrlPrefix() +
DistributedScheduler.getLockPath(segName);
}
}
[02/50] [abbrv] kylin git commit: KYLIN-2583 introduce DBCP as DataSource with cache logic
KYLIN-2583 introduce DBCP as DataSource with cache logic
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5e6ed11e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5e6ed11e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5e6ed11e
Branch: refs/heads/KYLIN-2624
Commit: 5e6ed11efd9a6a607fd1c8abf01f2c0d829c5e62
Parents: ca106b8
Author: Yifan Zhang
Authored: Fri May 5 09:46:28 2017 +0800
Committer: hongbin ma
Committed: Fri May 5 10:31:54 2017 +0800
--
.../org/apache/kylin/query/KylinTestBase.java | 4 +-
.../java/org/apache/kylin/query/QueryCli.java | 9 +-
.../org/apache/kylin/query/QueryDataSource.java | 162 +++
.../apache/kylin/query/QueryDataSourceTest.java | 86 ++
.../apache/kylin/rest/service/CacheService.java | 39 +
5 files changed, 256 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 2174094..fad7a94 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -49,7 +49,6 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
-import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.dbunit.DatabaseUnitException;
import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
@@ -659,8 +658,7 @@ public class KylinTestBase {
config = KylinConfig.getInstanceFromEnv();
//setup cube conn
-File olapTmp =
OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME,
config);
-cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" +
olapTmp.getAbsolutePath());
+cubeConnection =
QueryDataSource.create(ProjectInstance.DEFAULT_PROJECT_NAME,
config).getConnection();
//setup h2
h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" +
(h2InstanceCount++) + ";CACHE_SIZE=32072", "sa", "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/query/src/main/java/org/apache/kylin/query/QueryCli.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/QueryCli.java
b/query/src/main/java/org/apache/kylin/query/QueryCli.java
index d3f1521..5ced8e3 100644
--- a/query/src/main/java/org/apache/kylin/query/QueryCli.java
+++ b/query/src/main/java/org/apache/kylin/query/QueryCli.java
@@ -18,14 +18,11 @@
package org.apache.kylin.query;
-import java.io.File;
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
-import org.apache.calcite.jdbc.Driver;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -34,7 +31,6 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DBUtils;
-import org.apache.kylin.query.schema.OLAPSchemaFactory;
public class QueryCli {
@@ -55,14 +51,11 @@ public class QueryCli {
KylinConfig config =
KylinConfig.createInstanceFromUri(commandLine.getOptionValue(OPTION_METADATA.getOpt()));
String sql = commandLine.getOptionValue(OPTION_SQL.getOpt());
-Class.forName(Driver.class.getName());
-File olapTmp = OLAPSchemaFactory.createTempOLAPJson(null, config);
-
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try {
-conn = DriverManager.getConnection("jdbc:calcite:model=" +
olapTmp.getAbsolutePath());
+conn = QueryDataSource.create(null, config).getConnection();
stmt = conn.createStatement();
rs = stmt.executeQuery(sql);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
b/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
new file mode 100644
index 000..32feea6
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
@@ -0,0 +1,162 @
[05/50] [abbrv] kylin git commit: KYLIN-2578 Refactor DistributedLock
KYLIN-2578 Refactor DistributedLock
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7df407da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7df407da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7df407da
Branch: refs/heads/KYLIN-2624
Commit: 7df407da0cff32feaba4570698fdddcd86c4c48f
Parents: 783bb4e
Author: Yang Li
Authored: Sat May 6 22:33:10 2017 +0800
Committer: Yang Li
Committed: Sun May 7 14:51:53 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 8 +-
.../kylin/common/lock/DistributedLock.java | 61 +++-
.../common/lock/DistributedLockFactory.java | 43 +++
.../kylin/dict/GlobalDictionaryBuilder.java | 67 +
.../impl/threadpool/DistributedScheduler.java | 45 +--
.../kylin/job/lock/DistributedJobLock.java | 24 --
.../kylin/job/BaseTestDistributedScheduler.java | 26 +-
.../job/ITDistributedSchedulerBaseTest.java | 6 +-
.../job/ITDistributedSchedulerTakeOverTest.java | 2 +-
.../hbase/util/ZookeeperDistributedJobLock.java | 257 -
.../hbase/util/ZookeeperDistributedLock.java| 280 +++
.../storage/hbase/util/ZookeeperJobLock.java| 144 +++---
.../apache/kylin/storage/hdfs/LockManager.java | 8 +-
.../util/ITZookeeperDistributedLockTest.java| 254 +
14 files changed, 721 insertions(+), 504 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7df407da/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index eafdbbb..956518f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.lock.DistributedLock;
+import org.apache.kylin.common.lock.DistributedLockFactory;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.slf4j.Logger;
@@ -238,9 +238,9 @@ abstract public class KylinConfigBase implements
Serializable {
return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
}
-public DistributedLock getDistributedLock() {
-String clsName = getOptional("kylin.metadata.distributed-lock-impl",
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedJobLock");
-return (DistributedLock) ClassUtil.newInstance(clsName);
+public DistributedLockFactory getDistributedLockFactory() {
+String clsName = getOptional("kylin.metadata.distributed-lock-impl",
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
+return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}
//
http://git-wip-us.apache.org/repos/asf/kylin/blob/7df407da/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
b/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
index 9f04f05..e8844fd 100644
---
a/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
+++
b/core-common/src/main/java/org/apache/kylin/common/lock/DistributedLock.java
@@ -21,17 +21,66 @@ package org.apache.kylin.common.lock;
import java.io.Closeable;
import java.util.concurrent.Executor;
-public interface DistributedLock extends Closeable {
+/**
+ * A distributed lock. Every instance is owned by a client, on whose behalf
locks are acquired and/or released.
+ */
+public interface DistributedLock {
-boolean lockPath(String lockPath, String lockClient);
+/**
+ * Returns the client that owns this instance.
+ */
+String getClient();
+
+/**
+ * Acquire the lock at given path, non-blocking.
+ *
+ * @return If the lock is acquired or not.
+ */
+boolean lock(String lockPath);
+
+/**
+ * Acquire the lock at given path, block until given timeout.
+ *
+ * @return If the lock is acquired or not.
+ */
+boolean lock(String lockPath, long timeout);
-boolean isPathLocked(String lockPath);
+/**
+ * Returns if lock is available at given path.
+ */
+boolean isLocked(String lockPath);
+
+/**
+ * Returns if lock is available at given path.
+ */
+
[15/50] [abbrv] kylin git commit: KYLIN-2573 'kylin.sh stop' can't kill kylin's process by sometimes.
KYLIN-2573 'kylin.sh stop' can't kill kylin's process by sometimes.
Signed-off-by: Billy Liu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/901b10ce
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/901b10ce
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/901b10ce
Branch: refs/heads/KYLIN-2624
Commit: 901b10cee549a9c1e8d2df711a8c8d83789d7717
Parents: 42f7e7d
Author: mango77881 <12169...@qq.com>
Authored: Fri May 5 15:25:56 2017 +0800
Committer: Billy Liu
Committed: Tue May 9 13:47:32 2017 +0800
--
build/bin/kylin.sh | 41 ++---
1 file changed, 34 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/901b10ce/build/bin/kylin.sh
--
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index cd87436..d9e932e 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -131,20 +131,47 @@ then
if [ -f "${KYLIN_HOME}/pid" ]
then
PID=`cat $KYLIN_HOME/pid`
+WAIT_TIME=2
+LOOP_COUNTER=10
if ps -p $PID > /dev/null
then
- echo "Stopping Kylin: $PID"
- kill $PID
- rm ${KYLIN_HOME}/pid
- exit 0
+echo "Stopping Kylin: $PID"
+kill $PID
+
+for ((i=0; i<$LOOP_COUNTER; i++))
+do
+# wait to process stopped
+sleep $WAIT_TIME
+if ps -p $PID > /dev/null ; then
+echo "Stopping in progress. Will check after $WAIT_TIME
secs again..."
+continue;
+else
+break;
+fi
+done
+
+# if process is still around, use kill -9
+if ps -p $PID > /dev/null
+then
+echo "Initial kill failed, getting serious now..."
+kill -9 $PID
+sleep 1 #give kill -9 sometime to "kill"
+if ps -p $PID > /dev/null
+then
+ quit "Warning, even kill -9 failed, giving up! Sorry..."
+fi
+fi
+
+# process is killed , remove pid file
+rm -rf ${KYLIN_HOME}/pid
+echo "Kylin with pid ${PID} has been stopped."
+exit 0
else
- quit "Kylin is not running"
+ quit "Kylin with pid ${PID} is not running"
fi
-
else
quit "Kylin is not running"
fi
-
elif [ "$1" = "version" ]
then
retrieveDependency
kylin git commit: KYLIN-2624 fix IT with refactored DistributedLock
Repository: kylin
Updated Branches:
refs/heads/master f36372bc6 -> be32f7588
KYLIN-2624 fix IT with refactored DistributedLock
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be32f758
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be32f758
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be32f758
Branch: refs/heads/master
Commit: be32f75883c21bf32545d1b06158ad7cf36d676b
Parents: f36372b
Author: Yang Li
Authored: Fri May 19 22:02:38 2017 +0800
Committer: Yang Li
Committed: Fri May 19 22:02:38 2017 +0800
--
.../job/impl/threadpool/DistributedScheduler.java | 1 +
.../hbase/util/ZookeeperDistributedLock.java| 16
2 files changed, 13 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/be32f758/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index d01cb5a..8812dad 100644
---
a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++
b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -319,6 +319,7 @@ public class DistributedScheduler implements
Scheduler, Conn
}
return path;
}
+
@Override
public void shutdown() throws SchedulerException {
logger.info("Will shut down Job Engine ");
http://git-wip-us.apache.org/repos/asf/kylin/blob/be32f758/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
--
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
index d181d81..dc2a23e 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedLock.java
@@ -281,10 +281,18 @@ public class ZookeeperDistributedLock implements
DistributedLock, JobLock {
// normalize lock path
private String norm(String lockPath) {
-if (lockPath.startsWith(zkPathBase))
-return lockPath;
-else
-return zkPathBase + (lockPath.startsWith("/") ? "" : "/") +
lockPath;
+if (!lockPath.startsWith(zkPathBase))
+lockPath = zkPathBase + (lockPath.startsWith("/") ? "" : "/") +
lockPath;
+
+return dropDoubleSlash(lockPath);
+}
+
+public static String dropDoubleSlash(String path) {
+for (int n = Integer.MAX_VALUE; n > path.length();) {
+n = path.length();
+path = path.replace("//", "/");
+}
+return path;
}
//
[17/37] kylin git commit: KYLIN-2575 Experimental feature: Computed Column
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
--
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
index 221ffb6..cbf7a76 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
@@ -1,455 +1,627 @@
{
- "uuid" : "629ab7a8-3929-4dff-b59d-2100aadccd1a",
- "name" : "ci_left_join_cube",
- "model_name" : "ci_left_join_model",
- "description" : null,
- "dimensions" : [ {
-"name" : "CAL_DT",
-"table" : "TEST_CAL_DT",
-"column" : "{FK}",
-"derived" : [ "WEEK_BEG_DT" ]
- }, {
-"name" : "ORDER_ID",
-"table" : "TEST_KYLIN_FACT",
-"column" : "ORDER_ID"
- }, {
-"name" : "TEST_DATE_ENC",
-"table" : "TEST_ORDER",
-"column" : "TEST_DATE_ENC"
- }, {
-"name" : "TEST_TIME_ENC",
-"table" : "TEST_ORDER",
-"column" : "TEST_TIME_ENC"
- }, {
-"name" : "CATEGORY",
-"table" : "TEST_CATEGORY_GROUPINGS",
-"column" : "{FK}",
-"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE",
"UPD_USER" ]
- }, {
-"name" : "CATEGORY_HIERARCHY",
-"table" : "TEST_CATEGORY_GROUPINGS",
-"column" : "META_CATEG_NAME",
-"derived" : null
- }, {
-"name" : "CATEGORY_HIERARCHY",
-"table" : "TEST_CATEGORY_GROUPINGS",
-"column" : "CATEG_LVL2_NAME",
-"derived" : null
- }, {
-"name" : "CATEGORY_HIERARCHY",
-"table" : "TEST_CATEGORY_GROUPINGS",
-"column" : "CATEG_LVL3_NAME",
-"derived" : null
- }, {
-"name" : "LSTG_FORMAT_NAME",
-"table" : "TEST_KYLIN_FACT",
-"column" : "LSTG_FORMAT_NAME",
-"derived" : null
- }, {
-"name" : "SITE_ID",
-"table" : "TEST_SITES",
-"column" : "{FK}",
-"derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
-"name" : "SELLER_TYPE_CD",
-"table" : "TEST_SELLER_TYPE_DIM",
-"column" : "{FK}",
-"derived" : [ "SELLER_TYPE_DESC" ]
- }, {
-"name" : "SELLER_ID",
-"table" : "TEST_KYLIN_FACT",
-"column" : "SELLER_ID"
- }, {
-"name" : "SELLER_BUYER_LEVEL",
-"table" : "SELLER_ACCOUNT",
-"column" : "ACCOUNT_BUYER_LEVEL"
- }, {
-"name" : "SELLER_SELLER_LEVEL",
-"table" : "SELLER_ACCOUNT",
-"column" : "ACCOUNT_SELLER_LEVEL"
- }, {
-"name" : "SELLER_COUNTRY",
-"table" : "SELLER_ACCOUNT",
-"column" : "ACCOUNT_COUNTRY"
- }, {
-"name" : "SELLER_COUNTRY_NAME",
-"table" : "SELLER_COUNTRY",
-"column" : "NAME"
- }, {
-"name" : "BUYER_ID",
-"table" : "TEST_ORDER",
-"column" : "BUYER_ID"
- }, {
-"name" : "BUYER_BUYER_LEVEL",
-"table" : "BUYER_ACCOUNT",
-"column" : "ACCOUNT_BUYER_LEVEL"
- }, {
-"name" : "BUYER_SELLER_LEVEL",
-"table" : "BUYER_ACCOUNT",
-"column" : "ACCOUNT_SELLER_LEVEL"
- }, {
-"name" : "BUYER_COUNTRY",
-"table" : "BUYER_ACCOUNT",
-"column" : "ACCOUNT_COUNTRY"
- }, {
-"name" : "BUYER_COUNTRY_NAME",
-"table" : "BUYER_COUNTRY",
-"column" : "NAME"
- } ],
- "measures" : [ {
-"name" : "TRANS_CNT",
-"function" : {
- "expression" : "COUNT",
- "parameter" : {
-"type" : "constant",
-"value" : "1"
- },
- "returntype" : "bigint"
+ "uuid": "629ab7a8-3929-4dff-b59d-2100aadccd1a",
+ "name": "ci_left_join_cube",
+ "model_name": "ci_left_join_model",
+ "description": null,
+ "dimensions": [
+{
+ "name": "CAL_DT",
+ "table": "TEST_CAL_DT",
+ "column": "{FK}",
+ "derived": [
+"WEEK_BEG_DT"
+ ]
+},
+{
+ "name": "ORDER_ID",
+ "table": "TEST_KYLIN_FACT",
+ "column": "ORDER_ID"
+},
+{
+ "name": "TEST_DATE_ENC",
+ "table": "TEST_ORDER",
+ "column": "TEST_DATE_ENC"
+},
+{
+ "name": "TEST_TIME_ENC",
+ "table": "TEST_ORDER",
+ "column": "TEST_TIME_ENC"
+},
+{
+ "name": "CATEGORY",
+ "table": "TEST_CATEGORY_GROUPINGS",
+ "column": "{FK}",
+ "derived": [
+"USER_DEFINED_FIELD1",
+"USER_DEFINED_FIELD3",
+"UPD_DATE",
+"UPD_USER"
+ ]
+},
+{
+ "name": "CATEGORY_HIERARCHY",
+ "table": "TEST_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+},
+{
+ "name": "CATEGORY_HIERARCHY",
+ "table": "TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+},
+{
+ "name": "CATEGORY_HIERARCHY",
+ "table": "TEST_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
+},
+{
+ "name": "LSTG_FORMAT_NAME",
+ "table": "TEST_KYLIN_FACT",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+},
+{
+ "name": "SITE_ID",
+ "table": "TEST_SITES",
+ "col
[12/37] kylin git commit: KYLIN-2583 introduce DBCP as DataSource with cache logic
KYLIN-2583 introduce DBCP as DataSource with cache logic
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5e6ed11e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5e6ed11e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5e6ed11e
Branch: refs/heads/master
Commit: 5e6ed11efd9a6a607fd1c8abf01f2c0d829c5e62
Parents: ca106b8
Author: Yifan Zhang
Authored: Fri May 5 09:46:28 2017 +0800
Committer: hongbin ma
Committed: Fri May 5 10:31:54 2017 +0800
--
.../org/apache/kylin/query/KylinTestBase.java | 4 +-
.../java/org/apache/kylin/query/QueryCli.java | 9 +-
.../org/apache/kylin/query/QueryDataSource.java | 162 +++
.../apache/kylin/query/QueryDataSourceTest.java | 86 ++
.../apache/kylin/rest/service/CacheService.java | 39 +
5 files changed, 256 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
--
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 2174094..fad7a94 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -49,7 +49,6 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
-import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.dbunit.DatabaseUnitException;
import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
@@ -659,8 +658,7 @@ public class KylinTestBase {
config = KylinConfig.getInstanceFromEnv();
//setup cube conn
-File olapTmp =
OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME,
config);
-cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" +
olapTmp.getAbsolutePath());
+cubeConnection =
QueryDataSource.create(ProjectInstance.DEFAULT_PROJECT_NAME,
config).getConnection();
//setup h2
h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" +
(h2InstanceCount++) + ";CACHE_SIZE=32072", "sa", "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/query/src/main/java/org/apache/kylin/query/QueryCli.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/QueryCli.java
b/query/src/main/java/org/apache/kylin/query/QueryCli.java
index d3f1521..5ced8e3 100644
--- a/query/src/main/java/org/apache/kylin/query/QueryCli.java
+++ b/query/src/main/java/org/apache/kylin/query/QueryCli.java
@@ -18,14 +18,11 @@
package org.apache.kylin.query;
-import java.io.File;
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
-import org.apache.calcite.jdbc.Driver;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -34,7 +31,6 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DBUtils;
-import org.apache.kylin.query.schema.OLAPSchemaFactory;
public class QueryCli {
@@ -55,14 +51,11 @@ public class QueryCli {
KylinConfig config =
KylinConfig.createInstanceFromUri(commandLine.getOptionValue(OPTION_METADATA.getOpt()));
String sql = commandLine.getOptionValue(OPTION_SQL.getOpt());
-Class.forName(Driver.class.getName());
-File olapTmp = OLAPSchemaFactory.createTempOLAPJson(null, config);
-
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try {
-conn = DriverManager.getConnection("jdbc:calcite:model=" +
olapTmp.getAbsolutePath());
+conn = QueryDataSource.create(null, config).getConnection();
stmt = conn.createStatement();
rs = stmt.executeQuery(sql);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
--
diff --git a/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
b/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
new file mode 100644
index 000..32feea6
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/QueryDataSource.java
@@ -0,0 +1,162 @@
+/
[18/37] kylin git commit: KYLIN-2575 Experimental feature: Computed Column
KYLIN-2575 Experimental feature: Computed Column
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9a812d5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9a812d5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9a812d5f
Branch: refs/heads/master
Commit: 9a812d5fc917ddcdb736cd1c91fdfe9c933e603a
Parents: d6fdda2
Author: Hongbin Ma
Authored: Mon May 8 15:49:57 2017 +0800
Committer: liyang-gmt8
Committed: Mon May 8 16:01:48 2017 +0800
--
.../kylin/cube/CubeCapabilityChecker.java | 11 +-
.../java/org/apache/kylin/cube/JoinChecker.java | 60 -
.../org/apache/kylin/cube/cuboid/CuboidCLI.java |7 +
.../org/apache/kylin/cube/model/CubeDesc.java | 12 +-
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 23 +
.../model/validation/rule/FunctionRuleTest.java | 11 +-
.../org/apache/kylin/job/JoinedFlatTable.java |4 +-
.../apache/kylin/metadata/MetadataManager.java | 31 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 41 +-
.../metadata/model/ComputedColumnDesc.java | 95 ++
.../kylin/metadata/model/DataModelDesc.java | 81 +-
.../apache/kylin/metadata/model/JoinsTree.java |2 +-
.../kylin/metadata/model/PartitionDesc.java | 10 +-
.../apache/kylin/metadata/model/TableDesc.java | 35 +-
.../apache/kylin/metadata/model/TableRef.java | 14 +-
.../apache/kylin/metadata/model/TblColRef.java | 36 +-
.../kylin/metadata/project/ProjectL2Cache.java | 13 +-
.../kylin/metadata/model/DataModelDescTest.java | 15 +-
.../localmeta/cube_desc/ci_inner_join_cube.json | 1016 ++---
.../localmeta/cube_desc/ci_left_join_cube.json | 1036 ++
.../model_desc/ci_inner_join_model.json | 37 +-
.../model_desc/ci_left_join_model.json | 37 +-
.../test_case_data/sandbox/kylin.properties |2 +-
.../kylin/query/CompareQueryBySuffix.java | 37 +
.../kylin/query/ICompareQueryTranslator.java| 29 +
.../apache/kylin/query/ITKylinQueryTest.java|5 +
.../org/apache/kylin/query/KylinTestBase.java | 25 +-
.../query/sql_computedcolumn/query01.sql| 14 +
.../sql_computedcolumn/query01.sql.compare | 14 +
.../query/sql_computedcolumn/query02.sql| 21 +
.../sql_computedcolumn/query02.sql.compare | 21 +
.../query/sql_verifyCount/query01.sql.expected |2 +-
.../query/sql_verifyCount/query03.sql.expected |2 +-
.../query/sql_verifyCount/query04.sql.expected |2 +-
.../apache/kylin/query/schema/OLAPSchema.java |2 +-
.../kylin/rest/controller/ModelController.java |2 +-
.../kylin/rest/service/CacheServiceTest.java|1 -
.../kylin/rest/service/ModelServiceTest.java| 97 ++
.../apache/kylin/source/hive/SchemaChecker.java |2 +-
39 files changed, 1894 insertions(+), 1011 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index c45144b..20cb0a9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -52,12 +52,7 @@ public class CubeCapabilityChecker {
CapabilityResult result = new CapabilityResult();
result.capable = false;
-// match joins
-boolean isJoinMatch = JoinChecker.isJoinMatch(digest.joinDescs, cube);
-if (!isJoinMatch) {
-logger.info("Exclude cube " + cube.getName() + " because unmatched
joins");
-return result;
-}
+// match joins is ensured at model select
// dimensions & measures
Collection dimensionColumns = getDimensionColumns(digest);
@@ -106,12 +101,12 @@ public class CubeCapabilityChecker {
}
if (!unmatchedDimensions.isEmpty()) {
-logger.info("Exclude cube " + cube.getName() + " because unmatched
dimensions");
+logger.info("Exclude cube " + cube.getName() + " because unmatched
dimensions: " + unmatchedDimensions);
return result;
}
if (!unmatchedAggregations.isEmpty()) {
-logger.info("Exclude cube " + cube.getName() + " because unmatched
aggregations");
+logger.info("Exclude cube " + cube.getName() + " because unmatched
aggregations: " + unmatchedAggregations);
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
-
[28/37] kylin git commit: KYLIN-2596 Enable generating multiple streaming messages with one input message in streaming parser
KYLIN-2596 Enable generating multiple streaming messages with one input message
in streaming parser
* Minor, remove useless imports.
* Enable generating multiple streaming messages with one input message in
streaming parser
* Make MR input can generate multiple rows of date.
* For multiple rows, outputKV() should be called for each row.
* Try&catch for each row's data processing.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/edc4d4cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/edc4d4cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/edc4d4cc
Branch: refs/heads/master
Commit: edc4d4cc558473476b18d48f232635e44640c27a
Parents: 7c0038d
Author: nichunen
Authored: Fri May 12 14:55:14 2017 +0800
Committer: hongbin ma
Committed: Fri May 12 14:55:14 2017 +0800
--
.../java/org/apache/kylin/job/DeployUtil.java | 2 +-
.../kylin/common/util/StreamingMessage.java | 62 --
.../kylin/common/util/StreamingMessageRow.java | 62 ++
.../org/apache/kylin/engine/mr/IMRInput.java| 4 +-
.../mr/steps/FactDistinctColumnsMapper.java | 113 ++-
.../engine/mr/steps/HiveToBaseCuboidMapper.java | 15 ++-
.../engine/mr/steps/InMemCuboidMapper.java | 15 ++-
.../apache/kylin/source/hive/HiveMRInput.java | 6 +-
.../cardinality/ColumnCardinalityMapper.java| 25 ++--
.../apache/kylin/source/kafka/KafkaMRInput.java | 16 ++-
.../kylin/source/kafka/StreamingParser.java | 8 +-
.../source/kafka/TimedJsonStreamParser.java | 11 +-
.../source/kafka/TimedJsonStreamParserTest.java | 22 ++--
13 files changed, 194 insertions(+), 167 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/edc4d4cc/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
--
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index fdcd52c..077c056 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -166,7 +166,7 @@ public class DeployUtil {
TimedJsonStreamParser timedJsonStreamParser = new
TimedJsonStreamParser(tableColumns, null);
StringBuilder sb = new StringBuilder();
for (String json : data) {
-List rowColumns =
timedJsonStreamParser.parse(ByteBuffer.wrap(json.getBytes())).getData();
+List rowColumns =
timedJsonStreamParser.parse(ByteBuffer.wrap(json.getBytes())).get(0).getData();
sb.append(StringUtils.join(rowColumns, ","));
sb.append(System.getProperty("line.separator"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/edc4d4cc/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
b/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
deleted file mode 100644
index 981c8a8..000
---
a/core-common/src/main/java/org/apache/kylin/common/util/StreamingMessage.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- */
-public class StreamingMessage {
-
-private final List data;
-
-private long offset;
-
-private long timestamp;
-
-private Map params;
-
-public StreamingMessage(List data, long offset, long timestamp,
Map params) {
-this.data = data;
-this.offset = offset;
-this.timestamp = timestamp;
-this.params = params;
-}
-
-public final List getData() {
-return data;
-}
-
-public final long getOffset() {
-return offset;
-}
-
-public void setOffset(long offset) {
-this.offset = offset;
-}
-
-publ
[19/37] kylin git commit: KYLIN-2574 RawQueryLastHacker should group by all possible dimensions
KYLIN-2574 RawQueryLastHacker should group by all possible dimensions
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d6fdda2c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d6fdda2c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d6fdda2c
Branch: refs/heads/master
Commit: d6fdda2c24ac89ec72ff9663a36993f42a70960b
Parents: 1ba9af7
Author: Hongbin Ma
Authored: Fri Apr 28 15:03:43 2017 +0800
Committer: liyang-gmt8
Committed: Mon May 8 16:01:48 2017 +0800
--
.../main/java/org/apache/kylin/cube/RawQueryLastHacker.java| 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6fdda2c/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
--
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index 682e48c..d2e3a83 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -46,8 +46,8 @@ public class RawQueryLastHacker {
boolean isSelectAll = sqlDigest.allColumns.isEmpty() ||
sqlDigest.allColumns.equals(sqlDigest.filterColumns);
for (TblColRef col : cubeDesc.listAllColumns()) {
if (cubeDesc.listDimensionColumnsIncludingDerived().contains(col)
|| isSelectAll) {
-if (tupleInfo.hasColumn(col))
-sqlDigest.allColumns.add(col);
+//if (tupleInfo.hasColumn(col))
+sqlDigest.allColumns.add(col);
}
}
@@ -77,4 +77,4 @@ public class RawQueryLastHacker {
}
}
}
-}
\ No newline at end of file
+}
[04/37] kylin git commit: KYLIN-2577, fix hiveCmdBuilderTest UT
KYLIN-2577, fix hiveCmdBuilderTest UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/299b5841
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/299b5841
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/299b5841
Branch: refs/heads/master
Commit: 299b584151f75f475c61a88bd15167dbbd744386
Parents: ea5fcff
Author: Cheng Wang
Authored: Sat Apr 29 20:34:08 2017 +0800
Committer: Cheng Wang
Committed: Sat Apr 29 20:34:08 2017 +0800
--
.../org/apache/kylin/common/util/HiveCmdBuilder.java | 6 ++
.../org/apache/kylin/common/util/HiveCmdBuilderTest.java | 11 +--
2 files changed, 11 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/299b5841/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 9279973..255867a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -61,9 +61,8 @@ public class HiveCmdBuilder {
for (String statement : statements) {
buf.append(statement).append("\n");
}
-buf.append("\"").append(" \\").append("\n");
+buf.append("\"");
buf.append(parseProps());
-buf.append("\n");
break;
case BEELINE:
BufferedWriter bw = null;
@@ -110,11 +109,10 @@ public class HiveCmdBuilder {
private String parseProps() {
StringBuilder s = new StringBuilder();
for (Map.Entry prop : hiveConfProps.entrySet()) {
-s.append("--hiveconf ");
+s.append(" --hiveconf ");
s.append(prop.getKey());
s.append("=");
s.append(prop.getValue());
-s.append(" \\").append("\n");
}
return s.toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/299b5841/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index 3bc34b4..d847575 100644
---
a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++
b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -24,6 +24,8 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.junit.After;
@@ -48,12 +50,17 @@ public class HiveCmdBuilderTest {
public void testHiveCLI() {
System.setProperty("kylin.source.hive.client", "cli");
+Map hiveProps = new HashMap<>();
+hiveProps.put("hive.execution.engine", "mr");
+Map hivePropsOverwrite = new HashMap<>();
+hivePropsOverwrite.put("hive.execution.engine", "tez");
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
hiveCmdBuilder.addStatement("SHOW\n TABLES;");
-
-assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n
TABLES;\n\"", hiveCmdBuilder.build());
+hiveCmdBuilder.setHiveConfProps(hiveProps);
+hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite);
+assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n
TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
}
@Test
[14/37] kylin git commit: KYLIN-2588 fix ParameterDesc.equals()
KYLIN-2588 fix ParameterDesc.equals()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03969236
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03969236
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03969236
Branch: refs/heads/master
Commit: 0396923651cda8428b3022febcf013ff6d0457cf
Parents: 9a0598e
Author: lidongsjtu
Authored: Fri May 5 18:49:48 2017 +0800
Committer: hongbin ma
Committed: Fri May 5 20:56:52 2017 +0800
--
.../java/org/apache/kylin/metadata/model/ParameterDesc.java| 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/03969236/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 0cf31b1..930dc02 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -96,7 +96,7 @@ public class ParameterDesc implements Serializable {
void setValue(String value) {
this.value = value;
}
-
+
public TblColRef getColRef() {
return colRef;
}
@@ -112,7 +112,7 @@ public class ParameterDesc implements Serializable {
while (p != null) {
if (p.isColumnType())
all.add(p.getColRef());
-
+
p = p.nextParameter;
}
allColRefsIncludingNexts = all;
@@ -145,7 +145,7 @@ public class ParameterDesc implements Serializable {
if (p.isColumnType()) {
if (q.isColumnType() == false)
return false;
-if (this.getColRef().equals(that.getColRef()) == false)
+if (q.getColRef().equals(p.getColRef()) == false)
return false;
} else {
if (q.isColumnType() == true)
[11/37] kylin git commit: Support dropping jobs.
Support dropping jobs.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca106b82
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca106b82
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca106b82
Branch: refs/heads/master
Commit: ca106b824f50613385c98f8276bcbfaa04cb6cde
Parents: 8a4b6b9
Author: nichunen
Authored: Wed May 3 15:30:34 2017 +0800
Committer: hongbin ma
Committed: Thu May 4 22:32:55 2017 +0800
--
.../kylin/rest/controller/JobController.java| 23 +++
.../apache/kylin/rest/service/JobService.java | 6
webapp/app/js/controllers/job.js| 30
webapp/app/js/model/jobListModel.js | 1 +
webapp/app/js/services/jobs.js | 3 +-
webapp/app/partials/jobs/jobList.html | 4 +++
6 files changed, 66 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca106b82/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
index 6dc7887..7b1de37 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -28,7 +28,9 @@ import java.util.Map;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.constant.JobTimeFilterEnum;
+import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.request.JobListRequest;
import org.apache.kylin.rest.service.JobService;
import org.slf4j.Logger;
@@ -194,6 +196,27 @@ public class JobController extends BasicController {
}
}
+/**
+ * Drop a cube job
+ *
+ * @return
+ * @throws IOException
+ */
+@RequestMapping(value = "/{jobId}/drop", method = { RequestMethod.DELETE })
+@ResponseBody
+public JobInstance dropJob(@PathVariable String jobId) {
+JobInstance jobInstance = null;
+try {
+jobInstance = jobService.getJobInstance(jobId);
+jobService.dropJob(jobInstance);
+} catch (Exception e) {
+logger.error(e.getLocalizedMessage(), e);
+throw new InternalErrorException(e);
+}
+
+return jobInstance;
+}
+
public void setJobService(JobService jobService) {
this.jobService = jobService;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca106b82/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 4ba426e..56a148d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -321,6 +321,12 @@ public class JobService extends BasicService implements
InitializingBean {
return job;
}
+@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job,
'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job,
'MANAGEMENT')")
+public void dropJob(JobInstance job) throws IOException, JobException {
+cancelJob(job);
+getExecutableManager().deleteJob(job.getId());
+}
+
/**
* currently only support substring match
* @return
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca106b82/webapp/app/js/controllers/job.js
--
diff --git a/webapp/app/js/controllers/job.js b/webapp/app/js/controllers/job.js
index e6aba7e..abb3bed 100644
--- a/webapp/app/js/controllers/job.js
+++ b/webapp/app/js/controllers/job.js
@@ -215,6 +215,36 @@ KylinApp
});
}
+ $scope.drop = function (job) {
+SweetAlert.swal({
+ title: '',
+ text: 'Are you sure to drop the job?',
+ type: '',
+ showCancelButton: true,
+ confirmButtonColor: '#DD6B55',
+ confirmButtonText: "Yes",
+ closeOnConfirm: true
+}, function(isConfirm) {
+ if(isConfirm) {
+loadingRequest.show();
+JobService.drop({jobId: job.uuid}, {}, function (job) {
+ loadingRequest.hide();
+ SweetAlert.swal('Success!', 'Jo
[37/37] kylin git commit: Merge commit '87fedf75e5bbf61a33cf1b31720ad9ac55de6287'
Merge commit '87fedf75e5bbf61a33cf1b31720ad9ac55de6287'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f36372bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f36372bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f36372bc
Branch: refs/heads/master
Commit: f36372bc657b1edb312b36a071b077cde184eab8
Parents: 8ce972f 87fedf7
Author: Hongbin Ma
Authored: Fri May 19 17:55:26 2017 +0800
Committer: Hongbin Ma
Committed: Fri May 19 17:55:26 2017 +0800
--
.../rel/rules/OLAPJoinPushThroughJoinRule.java | 172
.../rel/rules/OLAPJoinPushThroughJoinRule2.java | 205 +
.../java/org/apache/calcite/tools/Programs.java | 438 +++
build/bin/find-hive-dependency.sh | 4 +-
build/bin/kylin.sh | 41 +-
build/bin/sample.sh | 24 +-
.../apache/kylin/common/KylinConfigBase.java| 8 +-
.../kylin/common/lock/DistributedLock.java | 63 ++-
.../common/lock/DistributedLockFactory.java | 43 ++
.../kylin/common/restclient/RestClient.java | 16 +-
.../org/apache/kylin/common/util/CheckUtil.java | 50 ++-
.../apache/kylin/cube/model/DimensionDesc.java | 3 +-
.../kylin/dict/BuiltInFunctionTransformer.java | 8 +-
.../kylin/dict/GlobalDictionaryBuilder.java | 71 +--
.../impl/threadpool/DistributedScheduler.java | 55 +--
.../kylin/job/lock/DistributedJobLock.java | 24 -
.../kylin/metadata/cachesync/Broadcaster.java | 66 ++-
.../metadata/filter/function/BuiltInMethod.java | 39 +-
.../metadata/filter/function/LikeMatchers.java | 141 ++
examples/sample_cube/create_sample_tables.sql | 10 +-
.../kylin/job/BaseTestDistributedScheduler.java | 35 +-
.../job/ITDistributedSchedulerBaseTest.java | 6 +-
.../job/ITDistributedSchedulerTakeOverTest.java | 2 +-
.../resources/query/sql_subquery/query17.sql| 17 +
.../resources/query/sql_subquery/query18.sql| 23 +
.../resources/query/sql_subquery/query19.sql| 20 +
.../resources/query/sql_subquery/query21.sql| 26 ++
.../resources/query/sql_subquery/query22.sql| 25 ++
.../resources/query/sql_subquery/query23.sql| 26 ++
.../resources/query/sql_subquery/query24.sql| 29 ++
.../resources/query/sql_subquery/query25.sql| 22 +
.../resources/query/sql_subquery/query26.sql| 16 +
.../resources/query/sql_subquery/query27.sql| 20 +
.../resources/query/sql_subquery/query28.sql| 21 +
.../query/sql_subquery/query29.sql.todo | 22 +
.../kylin/query/relnode/OLAPAggregateRel.java | 4 +-
.../apache/kylin/query/relnode/OLAPContext.java | 10 +
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 12 +-
.../kylin/query/relnode/OLAPProjectRel.java | 15 +-
.../rest/security/AuthoritiesPopulator.java | 20 +-
.../apache/kylin/rest/service/AclService.java | 1 -
.../kylin/rest/service/CacheServiceTest.java| 10 +-
.../kylin/storage/hbase/HBaseConnection.java| 30 +-
.../hbase/util/ZookeeperDistributedJobLock.java | 257 ---
.../hbase/util/ZookeeperDistributedLock.java| 307 +
.../storage/hbase/util/ZookeeperJobLock.java| 144 ++
.../apache/kylin/storage/hdfs/LockManager.java | 8 +-
.../util/ITZookeeperDistributedLockTest.java| 254 +++
.../apache/kylin/tool/MetadataCleanupJob.java | 14 +-
webapp/app/js/controllers/access.js | 1 +
webapp/app/js/controllers/admin.js | 12 +-
webapp/app/js/controllers/cubeMeasures.js | 2 +-
webapp/app/js/controllers/query.js | 4 +-
54 files changed, 2273 insertions(+), 625 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/build/bin/find-hive-dependency.sh
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/build/bin/sample.sh
--
diff --cc build/bin/sample.sh
index 2e13805,10900bb..b9c4ea3
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@@ -52,10 -62,10 +63,11 @@@ the
else
beeline_params=${beeline_params/${hive2_url}/${hive2_url}${sample_database}}
fi
- beeline ${hive_conf_properties} ${beeline_params} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
-beeline ${beeline_params} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
++
++beeline ${hive_conf_properties} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir}
${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || {
exit 1; }
else
-hive -e "CREATE DATABASE IF NOT EXISTS
[27/37] kylin git commit: KYLIN-2598 increase default kylin.query.translated-in-clause-max-size
KYLIN-2598 increase default kylin.query.translated-in-clause-max-size
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c0038db
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c0038db
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c0038db
Branch: refs/heads/master
Commit: 7c0038db4aab12c6fbd8f249b104a473e1d5a61c
Parents: b08d278
Author: Hongbin Ma
Authored: Fri May 12 13:11:01 2017 +0800
Committer: Dong Li
Committed: Fri May 12 13:12:08 2017 +0800
--
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c0038db/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 10209d7..b620924 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -845,7 +845,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public int getTranslatedInClauseMaxSize() {
-return
Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size",
"1024"));
+return
Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size",
String.valueOf(1024 * 1024)));
}
public int getLargeQueryThreshold() {
[10/37] kylin git commit: minor, fix typo
minor, fix typo
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8a4b6b96
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8a4b6b96
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8a4b6b96
Branch: refs/heads/master
Commit: 8a4b6b9611a25c8e0f8bf2ec99244a250d9dd91b
Parents: 32b8f2c
Author: Hongbin Ma
Authored: Thu May 4 16:58:09 2017 +0800
Committer: Dong Li
Committed: Thu May 4 17:00:22 2017 +0800
--
build/script/build.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a4b6b96/build/script/build.sh
--
diff --git a/build/script/build.sh b/build/script/build.sh
index 390a2c5..458c612 100755
--- a/build/script/build.sh
+++ b/build/script/build.sh
@@ -22,7 +22,7 @@ echo "package back-end"
dir=$(dirname ${0})
cd ${dir}/../..
-mvn clean intall -DskipTests $@ || { exit 1; }
+mvn clean install -DskipTests $@ || { exit 1; }
#package webapp
echo 'package front-end'
[15/37] kylin git commit: KYLIN-2428 more cleaning
KYLIN-2428 more cleaning
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8b277d5e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8b277d5e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8b277d5e
Branch: refs/heads/master
Commit: 8b277d5e492d8d54e6fd1f38eef7cf8d174f2b00
Parents: 0396923
Author: Hongbin Ma
Authored: Fri May 5 21:12:46 2017 +0800
Committer: liyang-gmt8
Committed: Sun May 7 21:22:28 2017 +0800
--
assembly/pom.xml| 4 --
core-common/pom.xml | 5 --
core-cube/pom.xml | 4 +-
core-dictionary/pom.xml | 5 --
engine-mr/pom.xml | 5 --
engine-spark/pom.xml| 8 +--
kylin-it/pom.xml| 5 --
pom.xml | 60 ---
server/pom.xml | 15 -
source-hive/pom.xml | 5 --
source-kafka/pom.xml| 7 +--
.../kylin/source/kafka/DateTimeParser.java | 62
storage-hbase/pom.xml | 4 +-
tool-assembly/pom.xml | 5 ++
14 files changed, 51 insertions(+), 143 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 121ccee..dae7152 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -207,10 +207,6 @@
${shadeBase}.org.apache.commons.codec
-
org.apache.commons.collections
-
${shadeBase}.org.apache.commons.collections
-
-
org.apache.commons.lang3
${shadeBase}.org.apache.commons.lang3
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/core-common/pom.xml
--
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 55f025f..8852743 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -64,11 +64,6 @@
-org.apache.hadoop
-hadoop-common
-provided
-
-
junit
junit
test
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/core-cube/pom.xml
--
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index cd382ef..930b6cd 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -44,12 +44,12 @@
kylin-core-dictionary
+
+
commons-collections
commons-collections
-
-
commons-lang
commons-lang
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/core-dictionary/pom.xml
--
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 09dde1e..c292982 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -41,11 +41,6 @@
-org.apache.hadoop
-hadoop-common
-provided
-
-
org.apache.kylin
kylin-core-common
test-jar
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/engine-mr/pom.xml
--
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 8aab79d..f2cedf6 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -55,11 +55,6 @@
org.apache.hadoop
-hadoop-common
-provided
-
-
-org.apache.hadoop
hadoop-hdfs
provided
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b277d5e/engine-spark/pom.xml
--
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index c5d23ce..93b6f9b 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -69,12 +69,6 @@
-org.apache.hadoop
-hadoop-common
-provided
-
-
-
org.apache.hbase
hbase-common
provided
@@ -120,4 +114,4 @@
-
\ No newline at end of file
+
[29/37] kylin git commit: KYLIN-2623 Move output(Hbase) code to outputside
KYLIN-2623 Move output(Hbase) code to outputside
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a38b02df
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a38b02df
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a38b02df
Branch: refs/heads/master
Commit: a38b02df0387541684aa1689e044927af650f1c7
Parents: edc4d4c
Author: Roger Shi
Authored: Tue May 16 13:50:23 2017 +0800
Committer: liyang-gmt8
Committed: Tue May 16 14:03:50 2017 +0800
--
.../org/apache/kylin/engine/mr/IMROutput2.java | 33 +-
.../engine/mr/common/AbstractHadoopJob.java | 6 +-
.../apache/kylin/engine/mr/steps/CuboidJob.java | 37 +++
.../kylin/engine/mr/steps/InMemCuboidJob.java | 55 ++
.../engine/mr/steps/LayerReducerNumSizing.java | 80 --
.../kylin/engine/mr/steps/MergeCuboidJob.java | 37 ---
.../engine/mr/steps/MergeCuboidMapper.java | 35 +-
.../kylin/engine/mr/steps/ReducerNumSizing.java | 106 +++
.../apache/kylin/source/hive/HiveMRInput.java | 2 -
.../apache/kylin/source/kafka/KafkaMRInput.java | 1 -
.../hbase/steps/HBaseMROutput2Transition.java | 97 +
11 files changed, 284 insertions(+), 205 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/a38b02df/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
index 603f207..69bba0a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput2.java
@@ -18,11 +18,14 @@
package org.apache.kylin.engine.mr;
+import java.util.List;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import java.util.List;
-
public interface IMROutput2 {
/** Return a helper to participate in batch cubing job flow. */
@@ -53,6 +56,19 @@ public interface IMROutput2 {
/** Add step that does any necessary clean up. */
public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow);
+
+public IMROutputFormat getOuputFormat();
+
+}
+
+public interface IMROutputFormat {
+
+/** Configure the InputFormat of given job. */
+public void configureJobInput(Job job, String input) throws Exception;
+
+/** Configure the OutputFormat of given job. */
+public void configureJobOutput(Job job, String output, CubeSegment
segment, int level) throws Exception;
+
}
/** Return a helper to participate in batch merge job flow. */
@@ -82,6 +98,19 @@ public interface IMROutput2 {
/** Add step that does any necessary clean up. */
public void addStepPhase3_Cleanup(DefaultChainedExecutable jobFlow);
+
+public IMRMergeOutputFormat getOuputFormat();
+}
+
+public interface IMRMergeOutputFormat {
+
+/** Configure the InputFormat of given job. */
+public void configureJobInput(Job job, String input) throws Exception;
+
+/** Configure the OutputFormat of given job. */
+public void configureJobOutput(Job job, String output, CubeSegment
segment) throws Exception;
+
+public CubeSegment findSourceSegment(FileSplit fileSplit, CubeInstance
cube);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a38b02df/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
--
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 44686d6..764cbdd 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -559,7 +559,7 @@ public abstract class AbstractHadoopJob extends Configured
implements Tool {
HadoopUtil.deletePath(conf, path);
}
-protected double getTotalMapInputMB() throws ClassNotFoundException,
IOException, InterruptedException, JobException {
+public static double getTotalMapInputMB(Job job) throws
ClassNotFoundException, IOException, InterruptedException, JobException {
if (job == null) {
throw new JobException("Job is null");
}
@@ -576,6 +576,10 @@ public abstract class AbstractHadoopJob extends Configured
implement
[34/37] kylin git commit: KYLIN-2526 refine find-hive-dependency.sh to support Ubuntu
KYLIN-2526 refine find-hive-dependency.sh to support Ubuntu
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0061900b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0061900b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0061900b
Branch: refs/heads/master
Commit: 0061900bb5bd34c8f0cc1c6e955d34bf34ca48e2
Parents: 5f52cea
Author: Cheng Wang
Authored: Thu May 18 11:04:47 2017 +0800
Committer: liyang-gmt8
Committed: Thu May 18 15:27:36 2017 +0800
--
build/bin/find-hive-dependency.sh | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0061900b/build/bin/find-hive-dependency.sh
--
diff --git a/build/bin/find-hive-dependency.sh
b/build/bin/find-hive-dependency.sh
index 02a47ef..c4c7523 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -100,10 +100,10 @@ fi
function checkFileExist()
{
-files=$1
+files=(`echo $1 | cut -d ":" -f 1- | sed 's/:/ /g'`)
misFiles=0
outputMissFiles=
-for file in ${files//:/ }
+for file in ${files}
do
let allFiles++
if [ ! -f "${file}" ]; then
@@ -111,8 +111,10 @@ function checkFileExist()
let misFiles++
fi
done
-ratio=`echo "scale=3; ${misFiles}/${allFiles}" | bc`
-[[ `echo "$ratio < 0.01" | bc ` -eq 1 ]] || quit "A couple of hive jars
can't be found: ${outputMisFiles}!"
+if [ 0 != ${misFiles} ]; then
+times=`expr ${allFiles} / ${misFiles}`
+[[ ${times} -gt 10 ]] || quit "A couple of hive jars can't be found:
${outputMisFiles}, please export HIVE_LIB='YOUR_LOCAL_HIVE_LIB'"
+fi
}
function validateDirectory()
[01/37] kylin git commit: after affects of KYLIN-2335
Repository: kylin
Updated Branches:
refs/heads/master 87fedf75e -> f36372bc6
after affects of KYLIN-2335
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d6098099
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d6098099
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d6098099
Branch: refs/heads/master
Commit: d609809939d8221ec37b8056abad359a58154f4b
Parents: 02a3d14
Author: Hongbin Ma
Authored: Fri Apr 28 13:26:57 2017 +0800
Committer: Hongbin Ma
Committed: Fri Apr 28 13:26:57 2017 +0800
--
.../rest/service/AclTableMigrationTool.java | 6 +-
.../apache/kylin/rest/service/UserService.java | 162 +--
.../kylin/rest/service/ServiceTestBase.java | 6 +-
3 files changed, 87 insertions(+), 87 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6098099/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
index da017b3..d147686 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -146,7 +146,7 @@ public class AclTableMigrationTool {
Result result = rs.next();
while (result != null) {
User user = hbaseRowToUser(result);
-UserInfo userInfo = convert(user);
+UserService.UserInfo userInfo = convert(user);
store.deleteResource(UserService.getId(userInfo.getUsername()));
store.putResource(UserService.getId(userInfo.getUsername()), userInfo, 0,
UserService.UserInfoSerializer.getInstance());
result = rs.next();
@@ -252,10 +252,10 @@ public class AclTableMigrationTool {
return newInfo;
}
-private UserInfo convert(User user) {
+private UserService.UserInfo convert(User user) {
if (user == null)
return null;
-UserInfo newInfo = new UserInfo();
+UserService.UserInfo newInfo = new UserService.UserInfo();
newInfo.setUsername(user.getUserName());
newInfo.setPassword(user.getPassword());
List authorities = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6098099/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 6a552fd..dd75ace 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -180,107 +180,107 @@ public class UserService implements UserDetailsManager {
}
}
-}
+public static class UserInfo extends RootPersistentEntity {
+@JsonProperty()
+private String username;
+@JsonProperty()
+private String password;
+@JsonProperty()
+private List authorities = new ArrayList<>();
+
+public UserInfo(String username, String password, List
authorities) {
+this.username = username;
+this.password = password;
+this.authorities = authorities;
+}
-class UserInfo extends RootPersistentEntity {
-@JsonProperty()
-private String username;
-@JsonProperty()
-private String password;
-@JsonProperty()
-private List authorities = new ArrayList<>();
-
-public UserInfo(String username, String password, List
authorities) {
-this.username = username;
-this.password = password;
-this.authorities = authorities;
-}
+public UserInfo(UserDetails user) {
+this.username = user.getUsername();
+this.password = user.getPassword();
+for (GrantedAuthority a : user.getAuthorities()) {
+this.authorities.add(a.getAuthority());
+}
+}
-public UserInfo(UserDetails user) {
-this.username = user.getUsername();
-this.password = user.getPassword();
-for (GrantedAuthority a : user.getAuthorities()) {
-this.authorities.add(a.getAuthority());
+public UserInfo() {
}
-}
-public UserInfo() {
-}
+public String getUsername() {
+return username;
+}
-public String getUsernam
[09/37] kylin git commit: minor, cleanup unused logs
minor, cleanup unused logs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32b8f2c7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32b8f2c7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32b8f2c7
Branch: refs/heads/master
Commit: 32b8f2c7fe7e4179dec9ce1ccafac917dbe2eb27
Parents: 0e0c970
Author: lidongsjtu
Authored: Wed May 3 19:45:10 2017 +0800
Committer: hongbin ma
Committed: Wed May 3 20:00:29 2017 +0800
--
.../main/java/org/apache/kylin/common/KylinConfig.java | 2 +-
.../apache/kylin/job/execution/AbstractExecutable.java | 4 ++--
.../apache/kylin/dimension/DimensionEncodingFactory.java | 2 --
.../metadata/filter/BuiltInFunctionTupleFilter.java | 2 +-
.../apache/kylin/metadata/project/ProjectL2Cache.java| 2 +-
.../kylin/engine/mr/common/HadoopShellExecutable.java| 3 +--
.../org/apache/kylin/rest/service/DiagnosisService.java | 11 +++
.../java/org/apache/kylin/tool/CubeMetaExtractor.java| 2 +-
8 files changed, 10 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/32b8f2c7/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index c00054f..04af9f5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -232,7 +232,7 @@ public class KylinConfig extends KylinConfigBase {
return getKylinPropertiesFile(kylinConfHome);
}
-logger.warn("KYLIN_CONF property was not set, will seek KYLIN_HOME env
variable");
+logger.debug("KYLIN_CONF property was not set, will seek KYLIN_HOME
env variable");
String kylinHome = getKylinHome();
if (StringUtils.isEmpty(kylinHome))
http://git-wip-us.apache.org/repos/asf/kylin/blob/32b8f2c7/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
--
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index cd9b033..1f1be41 100644
---
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -255,7 +255,7 @@ public abstract class AbstractExecutable implements
Executable, Idempotent {
try {
List users = getAllNofifyUsers(config);
if (users.isEmpty()) {
-logger.warn("no need to send email, user list is empty");
+logger.debug("no need to send email, user list is empty");
return;
}
final Pair email = formatNotifications(context,
state);
@@ -293,7 +293,7 @@ public abstract class AbstractExecutable implements
Executable, Idempotent {
try {
List users = getAllNofifyUsers(config);
if (users.isEmpty()) {
-logger.warn("no need to send email, user list is empty");
+logger.debug("no need to send email, user list is empty");
return;
}
doSendMail(config, users, email);
http://git-wip-us.apache.org/repos/asf/kylin/blob/32b8f2c7/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 3eddba7..8f1fcd6 100644
---
a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -18,7 +18,6 @@
package org.apache.kylin.dimension;
-import java.util.Arrays;
import java.util.Map;
import javax.annotation.Nullable;
@@ -54,7 +53,6 @@ public abstract class DimensionEncodingFactory {
* Create a DimensionEncoding instance, with inputs corresponding to
RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs.
*/
public static DimensionEncoding create(String encodingName, String[] args,
int version) {
-logger.debug("Encoding Name : {}, args : {}, version {}",
encodingName, Arrays.toString(args), version);
if (factoryMap == null)
initFactoryMap();
http://git-wip-us.apache.org/repos/asf/kylin/blob/32b8f2c7/core-metadata/src/m
[25/37] kylin git commit: minor, fix csv export garbled issue
minor, fix csv export garbled issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/949237dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/949237dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/949237dd
Branch: refs/heads/master
Commit: 949237ddca0ab24bf8121ccc002d802df678761c
Parents: a059064
Author: Hongbin Ma
Authored: Thu May 11 17:29:14 2017 +0800
Committer: Dong Li
Committed: Fri May 12 12:46:35 2017 +0800
--
.../java/org/apache/kylin/rest/controller/QueryController.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/949237dd/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index c5f896d..99447a1 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -103,7 +103,7 @@ public class QueryController extends BasicController {
@ResponseBody
public void downloadQueryResult(@PathVariable String format, SQLRequest
sqlRequest, HttpServletResponse response) {
SQLResponse result = queryService.doQueryWithCache(sqlRequest);
-response.setContentType("text/" + format + ";charset=utf-8");
+response.setContentType("text/" + format + ";charset=ansi");
response.setHeader("Content-Disposition", "attachment;
filename=\"result." + format + "\"");
ICsvListWriter csvWriter = null;
[06/37] kylin git commit: KYLIN-2535 minor, refactor some beans in acl service and user service
KYLIN-2535 minor, refactor some beans in acl service and user service
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2b6a91f1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2b6a91f1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2b6a91f1
Branch: refs/heads/master
Commit: 2b6a91f14e791347f19bb1a7811c691fb1b6afb0
Parents: 0b51f7b
Author: FAN XIE
Authored: Tue May 2 13:33:47 2017 +0800
Committer: hongbin ma
Committed: Tue May 2 13:33:47 2017 +0800
--
.../hbase/ITAclTableMigrationToolTest.java | 9 +-
.../org/apache/kylin/rest/service/AceInfo.java | 55 +
.../apache/kylin/rest/service/AclService.java | 95 ---
.../rest/service/AclTableMigrationTool.java | 53 ++---
.../kylin/rest/service/DomainObjectInfo.java| 52
.../kylin/rest/service/LegacyAclService.java| 118 ++-
.../kylin/rest/service/LegacyUserService.java | 51
.../org/apache/kylin/rest/service/SidInfo.java | 61 ++
8 files changed, 199 insertions(+), 295 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/2b6a91f1/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
--
diff --git
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
index 35535e5..65d5b52 100644
---
a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
+++
b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITAclTableMigrationToolTest.java
@@ -45,6 +45,7 @@ import org.apache.kylin.rest.security.AclHBaseStorage;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.AclTableMigrationTool;
import org.apache.kylin.rest.service.LegacyUserService;
+import org.apache.kylin.rest.service.UserGrantedAuthority;
import org.apache.kylin.rest.service.UserService;
import org.apache.kylin.rest.util.Serializer;
import org.junit.After;
@@ -70,7 +71,7 @@ public class ITAclTableMigrationToolTest extends
HBaseMetadataTestCase {
private TableName userTable = TableName.valueOf(STORE_WITH_OLD_TABLE +
AclHBaseStorage.USER_TABLE_NAME);
-private Serializer ugaSerializer
= new
Serializer(LegacyUserService.UserGrantedAuthority[].class);
+private Serializer ugaSerializer = new
Serializer(UserGrantedAuthority[].class);
private AclTableMigrationTool aclTableMigrationJob;
@@ -192,13 +193,13 @@ public class ITAclTableMigrationToolTest extends
HBaseMetadataTestCase {
if (authorities == null)
authorities = Collections.emptyList();
-LegacyUserService.UserGrantedAuthority[] serializing = new
LegacyUserService.UserGrantedAuthority[authorities.size() + 1];
+UserGrantedAuthority[] serializing = new
UserGrantedAuthority[authorities.size() + 1];
// password is stored as the [0] authority
-serializing[0] = new
LegacyUserService.UserGrantedAuthority(LegacyUserService.PWD_PREFIX +
"password");
+serializing[0] = new UserGrantedAuthority(LegacyUserService.PWD_PREFIX
+ "password");
int i = 1;
for (GrantedAuthority a : authorities) {
-serializing[i++] = new
LegacyUserService.UserGrantedAuthority(a.getAuthority());
+serializing[i++] = new UserGrantedAuthority(a.getAuthority());
}
byte[] value = ugaSerializer.serialize(serializing);
http://git-wip-us.apache.org/repos/asf/kylin/blob/2b6a91f1/server-base/src/main/java/org/apache/kylin/rest/service/AceInfo.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/AceInfo.java
b/server-base/src/main/java/org/apache/kylin/rest/service/AceInfo.java
new file mode 100644
index 000..0be1019
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AceInfo.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
[35/37] kylin git commit: KYLIN-2624 introduce StorageURL and set timeout for metadata access
KYLIN-2624 introduce StorageURL and set timeout for metadata access
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2dfc5bb3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2dfc5bb3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2dfc5bb3
Branch: refs/heads/master
Commit: 2dfc5bb36a00245b95deeebb5bb3bca9c353c934
Parents: 7c152c5
Author: Li Yang
Authored: Thu May 18 14:16:17 2017 +0800
Committer: nichunen
Committed: Fri May 19 13:52:53 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 30 ++-
.../org/apache/kylin/common/StorageURL.java | 183 +++
.../common/persistence/FileResourceStore.java | 2 +-
.../apache/kylin/common/KylinConfigTest.java| 9 +-
.../org/apache/kylin/common/StorageURLTest.java | 103 +++
.../persistence/LocalFileResourceStoreTest.java | 3 +-
.../common/persistence/ResourceStoreTest.java | 18 +-
.../common/util/LocalFileMetadataTestCase.java | 7 +
.../apache/kylin/dict/DictionaryManager.java| 18 +-
.../dict/global/AppendTrieDictionaryTest.java | 5 +-
.../impl/threadpool/DistributedScheduler.java | 11 +-
.../apache/kylin/storage/StorageContext.java| 7 +-
.../kylin/engine/mr/steps/CubeReducerTest.java | 2 +-
.../engine/mr/steps/NDCuboidMapperTest.java | 2 +-
.../kylin/job/BaseTestDistributedScheduler.java | 5 +-
.../kylin/storage/hbase/ITStorageTest.java | 3 +-
.../apache/kylin/query/schema/OLAPSchema.java | 5 +-
.../rest/security/MockAclHBaseStorage.java | 11 +-
.../rest/security/RealAclHBaseStorage.java | 15 +-
.../apache/kylin/rest/service/QueryService.java | 11 +-
.../kylin/storage/hbase/HBaseConnection.java| 51 +++---
.../kylin/storage/hbase/HBaseResourceStore.java | 46 +++--
.../storage/hbase/util/CubeMigrationCLI.java| 20 +-
.../hbase/util/GridTableHBaseBenchmark.java | 3 +-
.../kylin/storage/hdfs/HDFSResourceStore.java | 50 ++---
.../hbase/steps/CubeHFileMapper2Test.java | 4 +-
.../storage/hbase/steps/MockupMapContext.java | 2 +-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 20 +-
28 files changed, 461 insertions(+), 185 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfc5bb3/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b620924..caef95e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -169,7 +169,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String toString() {
-return getMetadataUrl();
+return getMetadataUrl().toString();
}
//
@@ -205,8 +205,8 @@ abstract public class KylinConfigBase implements
Serializable {
// METADATA
//
-public String getMetadataUrl() {
-return getOptional("kylin.metadata.url");
+public StorageURL getMetadataUrl() {
+return StorageURL.valueOf(getOptional("kylin.metadata.url", ""));
}
// for test only
@@ -215,14 +215,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String getMetadataUrlPrefix() {
-String metadataUrl = getMetadataUrl();
-String defaultPrefix = "kylin_metadata";
-
-int cut = metadataUrl.lastIndexOf('@');
-if (cut > 0) {
-return metadataUrl.substring(0, cut);
-}
-return defaultPrefix;
+return getMetadataUrl().getIdentifier();
}
public String[] getRealizationProviders() {
@@ -558,8 +551,14 @@ abstract public class KylinConfigBase implements
Serializable {
return Integer.parseInt(getOptional("kylin.storage.default", "2"));
}
-public String getStorageUrl() {
-return getOptional("kylin.storage.url");
+public StorageURL getStorageUrl() {
+String url = getOptional("kylin.storage.url", "default@hbase");
+
+// for backward compatibility
+if ("hbase".equals(url))
+url = "default@hbase";
+
+return StorageURL.valueOf(url);
}
public String getHBaseClusterFs() {
@@ -987,10 +986,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public String getResourceStoreImpl() {
-String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
-
[26/37] kylin git commit: minor, remove useless scripts
minor, remove useless scripts
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b08d278e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b08d278e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b08d278e
Branch: refs/heads/master
Commit: b08d278e5ed688457758eee43aff4f479fd0fb72
Parents: 949237d
Author: Hongbin Ma
Authored: Fri May 12 12:10:21 2017 +0800
Committer: Dong Li
Committed: Fri May 12 12:46:35 2017 +0800
--
build/script/package_diag.sh | 77 -
build/script/package_libs.sh | 100 --
2 files changed, 177 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/b08d278e/build/script/package_diag.sh
--
diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh
deleted file mode 100755
index 5d163e3..000
--- a/build/script/package_diag.sh
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-echo "Checking maven..."
-
-if [ -z "$(command -v mvn)" ]
-then
-echo "Please install maven first so that Kylin packaging can proceed"
-exit 1
-else
-echo "maven check passed"
-fi
-
-echo "Checking git..."
-
-if [ -z "$(command -v git)" ]
-then
-echo "Please install git first so that Kylin packaging can proceed"
-exit 1
-else
-echo "git check passed"
-fi
-
-dir=$(dirname ${0})
-cd ${dir}/../..
-version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate
-Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' `
-
-if [ "$version" == "" ];then
- echo "Failed to identify kylin version (current: `pwd`)"
- exit 1
-fi
-
-echo "kylin version: ${version}"
-
-echo "package libraries"
-mvn clean install -DskipTests || { exit 1; }
-
-echo "copy libraries"
-rm -rf build/tool
-mkdir build/tool
-cp tool/target/kylin-tool-${version}-assembly.jar
build/tool/kylin-tool-${version}.jar
-# Copied file becomes 000 for some env (e.g. my Cygwin)
-chmod 644 build/tool/kylin-tool-${version}.jar
-
-echo 'package tar.gz'
-package_name=apache-kylin-${version}-diag
-cd build/
-rm -rf ${package_name}
-mkdir ${package_name}
-mkdir ${package_name}/bin
-cp -r tool ${package_name}
-cp bin/diag.sh ${package_name}/bin/
-rm -rf tool
-find ${package_name} -type d -exec chmod 755 {} \;
-find ${package_name} -type f -exec chmod 644 {} \;
-find ${package_name} -type f -name "*.sh" -exec chmod 755 {} \;
-mkdir -p ../dist
-tar -cvzf ../dist/${package_name}.tar.gz ${package_name}
-rm -rf ${package_name}
-
-echo "Library package ready: dist/${package_name}.tar.gz"
http://git-wip-us.apache.org/repos/asf/kylin/blob/b08d278e/build/script/package_libs.sh
--
diff --git a/build/script/package_libs.sh b/build/script/package_libs.sh
deleted file mode 100755
index 7ef6b5d..000
--- a/build/script/package_libs.sh
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-#http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-echo "Checking maven..."
-
-if [ -z "$(command -v mvn)" ]
-then
-echo "Please install maven first so that Kylin packaging can proceed"
-exit 1
-else
-echo "maven check passed"
-fi
-
-echo "Checking git..."
-
-if [ -z "$(com
[32/37] kylin git commit: KYLIN-2635 fix IT
KYLIN-2635 fix IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f52cea4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f52cea4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f52cea4
Branch: refs/heads/master
Commit: 5f52cea4f7ffbb0fa885df650803156cd1daa5d2
Parents: ea5caba
Author: Hongbin Ma
Authored: Wed May 17 21:26:09 2017 +0800
Committer: Roger Shi
Committed: Thu May 18 11:02:49 2017 +0800
--
.../org/apache/kylin/metadata/filter/CompareTupleFilter.java | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f52cea4/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 2771250..c091c7c 100755
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -167,7 +167,13 @@ public class CompareTupleFilter extends TupleFilter {
return true;
else
return false;
+} else {
+if (operator == FilterOperatorEnum.ISNOTNULL)
+return true;
+else if (operator == FilterOperatorEnum.ISNULL)
+return false;
}
+
if (cs.isNull(firstCondValue)) {
return false;
}
[22/37] kylin git commit: KYLIN-2599 select * in subquery fail due to bug in hackSelectStar
KYLIN-2599 select * in subquery fail due to bug in hackSelectStar
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3d81832
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3d81832
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3d81832
Branch: refs/heads/master
Commit: c3d81832f796ab281526d3ac1cd97b11eb986a92
Parents: 9dc6553
Author: Hongbin Ma
Authored: Tue May 9 21:42:47 2017 +0800
Committer: liyang-gmt8
Committed: Wed May 10 09:41:40 2017 +0800
--
.../calcite/sql2rel/SqlToRelConverter.java | 11 +
.../src/test/resources/query/sql/query105.sql | 26
2 files changed, 37 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3d81832/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
--
diff --git
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index c199c31..a520aa6 100644
---
a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++
b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -47,6 +47,7 @@ import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.core.Sample;
import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.core.Uncollect;
import org.apache.calcite.rel.logical.LogicalAggregate;
import org.apache.calcite.rel.logical.LogicalCorrelate;
@@ -611,6 +612,16 @@ public class SqlToRelConverter {
} else {
return root;
}
+
+RelNode tableScanOrJoin = null;
+RelNode rootProjInput = rootPrj.getInput(0) instanceof LogicalFilter ?
rootPrj.getInput(0).getInput(0): rootPrj.getInput(0);
+if(rootProjInput instanceof TableScan || rootProjInput instanceof
LogicalJoin) {
+tableScanOrJoin = rootProjInput;
+}
+if(tableScanOrJoin == null) {
+return root;
+}
+
//
RelNode input = rootPrj.getInput();
//if (!(//
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3d81832/kylin-it/src/test/resources/query/sql/query105.sql
--
diff --git a/kylin-it/src/test/resources/query/sql/query105.sql
b/kylin-it/src/test/resources/query/sql/query105.sql
new file mode 100644
index 000..a467b65
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query105.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select fact.lstg_format_name from
+
+ (select * from test_kylin_fact where cal_dt > date'2010-01-01' ) as fact
+
+ group by fact.lstg_format_name
+
+ order by CASE WHEN fact.lstg_format_name IS NULL THEN 'sdf' ELSE
fact.lstg_format_name END
+
[36/37] kylin git commit: KAP-733 minor UT fix
KAP-733 minor UT fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ce972fd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ce972fd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ce972fd
Branch: refs/heads/master
Commit: 8ce972fd6475e923af2a5e525d8ddb698d0dbadf
Parents: 2dfc5bb
Author: Li Yang
Authored: Fri May 19 15:50:32 2017 +0800
Committer: liyang-gmt8
Committed: Fri May 19 17:25:55 2017 +0800
--
.../apache/kylin/common/util/LocalFileMetadataTestCase.java| 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ce972fd/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
--
diff --git
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index e5e4f8b..a44a55d 100644
---
a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++
b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -81,7 +81,11 @@ public class LocalFileMetadataTestCase extends
AbstractKylinTestCase {
String dir =
KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
if (dir.startsWith("file://"))
dir = dir.substring("file://".length());
-return dir;
+try {
+return new File(dir).getCanonicalPath();
+} catch (IOException e) {
+throw new RuntimeException(e);
+}
}
protected ResourceStore getStore() {
[08/37] kylin git commit: change all mvn clean package back to mvn clean install
change all mvn clean package back to mvn clean install
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0e0c970a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0e0c970a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0e0c970a
Branch: refs/heads/master
Commit: 0e0c970aadb17d9abec7166a3ab87b0c5a783d65
Parents: 4c58b98
Author: Hongbin Ma
Authored: Wed May 3 13:46:50 2017 +0800
Committer: Dong Li
Committed: Wed May 3 13:51:55 2017 +0800
--
build/script/build.sh | 2 +-
build/script/package_diag.sh| 2 +-
build/script/package_libs.sh| 2 +-
dev-support/sync_hbase_cdh_branches.sh | 2 +-
dev-support/test_all_against_hdp_2_4_0_0_169.sh | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e0c970a/build/script/build.sh
--
diff --git a/build/script/build.sh b/build/script/build.sh
index 6b1d767..390a2c5 100755
--- a/build/script/build.sh
+++ b/build/script/build.sh
@@ -22,7 +22,7 @@ echo "package back-end"
dir=$(dirname ${0})
cd ${dir}/../..
-mvn clean package -DskipTests $@ || { exit 1; }
+mvn clean intall -DskipTests $@ || { exit 1; }
#package webapp
echo 'package front-end'
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e0c970a/build/script/package_diag.sh
--
diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh
index 5a41f63..5d163e3 100755
--- a/build/script/package_diag.sh
+++ b/build/script/package_diag.sh
@@ -49,7 +49,7 @@ fi
echo "kylin version: ${version}"
echo "package libraries"
-mvn clean package -DskipTests || { exit 1; }
+mvn clean install -DskipTests || { exit 1; }
echo "copy libraries"
rm -rf build/tool
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e0c970a/build/script/package_libs.sh
--
diff --git a/build/script/package_libs.sh b/build/script/package_libs.sh
index 01e52bc..7ef6b5d 100755
--- a/build/script/package_libs.sh
+++ b/build/script/package_libs.sh
@@ -70,7 +70,7 @@ EOF
git rev-parse HEAD >> build/commit_SHA1
echo "package libraries"
-mvn clean package -DskipTests || { exit 1; }
+mvn clean install -DskipTests || { exit 1; }
echo "copy libraries"
rm -rf build/lib
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e0c970a/dev-support/sync_hbase_cdh_branches.sh
--
diff --git a/dev-support/sync_hbase_cdh_branches.sh
b/dev-support/sync_hbase_cdh_branches.sh
index 340c570..fba4fca 100755
--- a/dev-support/sync_hbase_cdh_branches.sh
+++ b/dev-support/sync_hbase_cdh_branches.sh
@@ -45,7 +45,7 @@ git checkout -b tmp
git reset apache/$base --hard
git am -3 --ignore-whitespace
0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
-mvn clean package -DskipTests
+mvn clean install -DskipTests
git push apache tmp:$base-hbase0.98 -f
rm 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e0c970a/dev-support/test_all_against_hdp_2_4_0_0_169.sh
--
diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh
b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
index 26ca825..2a3d24b 100755
--- a/dev-support/test_all_against_hdp_2_4_0_0_169.sh
+++ b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
@@ -21,5 +21,5 @@ dir=$(dirname ${0})
cd ${dir}
cd ..
-mvn clean package -DskipTests 2>&1 | tee mci.log
+mvn clean install -DskipTests 2>&1 | tee mci.log
mvn verify -Dhdp.version=${HDP_VERSION:-"2.4.0.0-169"} -fae 2>&1 | tee
mvnverify.log
[03/37] kylin git commit: KYLIN-2577, improve hive configurations during cubing steps
KYLIN-2577, improve hive configurations during cubing steps
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea5fcff8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea5fcff8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea5fcff8
Branch: refs/heads/master
Commit: ea5fcff8397026de315324bf1797b62944591908
Parents: 4c18526
Author: Cheng Wang
Authored: Fri Apr 28 17:52:58 2017 +0800
Committer: nichunen
Committed: Fri Apr 28 18:41:52 2017 +0800
--
build/bin/load-hive-conf.sh | 15
build/bin/sample.sh | 9 ++-
.../kylin/common/util/HiveCmdBuilder.java | 29 ++-
.../org/apache/kylin/job/JoinedFlatTable.java | 37 ++---
.../kylin/job/engine/JobEngineConfig.java | 32 +++-
.../source/hive/CreateFlatHiveTableStep.java| 4 +
.../apache/kylin/source/hive/HiveMRInput.java | 8 +-
.../org/apache/kylin/tool/HiveConfigCLI.java| 81
8 files changed, 173 insertions(+), 42 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea5fcff8/build/bin/load-hive-conf.sh
--
diff --git a/build/bin/load-hive-conf.sh b/build/bin/load-hive-conf.sh
new file mode 100644
index 000..a5046f0
--- /dev/null
+++ b/build/bin/load-hive-conf.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Kyligence Inc. License
+
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
+# source me
+
+hive_conf_dir="${KYLIN_HOME}/conf/kylin_hive_conf.xml"
+hive_conf_prop="${KYLIN_HOME}/logs/hive_props"
+rm -rf ${hive_conf_prop}
+export ENABLE_CHECK_ENV=false
+${dir}/kylin.sh org.apache.kylin.tool.HiveConfigCLI ${hive_conf_dir}
${hive_conf_prop}
+[[ 0 == $? ]] || quit "Error, can not parse ${hive_conf_dir} and can not apply
it to hive relevant check."
+hive_conf_properties=`cat ${hive_conf_prop}`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea5fcff8/build/bin/sample.sh
--
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 617b671..2e13805 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -19,6 +19,7 @@
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/find-hadoop-conf-dir.sh
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/load-hive-conf.sh
source ${dir}/check-env.sh "if-not-yet"
job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
@@ -43,7 +44,7 @@ echo "Going to create sample tables in hive to database
"$sample_database" by "$
if [ "${hive_client_mode}" == "beeline" ]
then
beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh
kylin.source.hive.beeline-params`
-beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS
"$sample_database
+beeline ${hive_conf_properties} ${beeline_params} -e "CREATE DATABASE IF
NOT EXISTS "$sample_database
hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\/\)'`
if [ -z ${hive2_url} ]; then
hive2_url=`expr match "${beeline_params}"
'.*\(hive2:.*:[0-9]\{4,6\}\)'`
@@ -51,10 +52,10 @@ then
else
beeline_params=${beeline_params/${hive2_url}/${hive2_url}${sample_database}}
fi
-beeline ${beeline_params} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
+beeline ${hive_conf_properties} ${beeline_params} -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
else
-hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database
-hive --database $sample_database -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
+hive ${hive_conf_properties} -e "CREATE DATABASE IF NOT EXISTS
"$sample_database
+hive ${hive_conf_properties} --database $sample_database -f
${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
fi
echo "Sample hive tables are created successfully; Going to create sample
cube..."
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea5fcff8/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 855ee48..9279973 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -24,6 +24,8 @@ import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.i
[02/37] kylin git commit: KYLIN-2535 minor, bug fix
KYLIN-2535 minor, bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c18526c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c18526c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c18526c
Branch: refs/heads/master
Commit: 4c18526cff0c8437f3724fb32833636050bb7fe1
Parents: d609809
Author: xiefan46 <958034...@qq.com>
Authored: Fri Apr 28 18:06:29 2017 +0800
Committer: hongbin ma
Committed: Fri Apr 28 18:09:53 2017 +0800
--
.../rest/service/AclTableMigrationTool.java | 6 +-
.../rest/service/UserGrantedAuthority.java | 74 +
.../org/apache/kylin/rest/service/UserInfo.java | 79 +
.../apache/kylin/rest/service/UserService.java | 111 +--
.../kylin/rest/service/ServiceTestBase.java | 6 +-
5 files changed, 164 insertions(+), 112 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c18526c/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
index d147686..da017b3 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -146,7 +146,7 @@ public class AclTableMigrationTool {
Result result = rs.next();
while (result != null) {
User user = hbaseRowToUser(result);
-UserService.UserInfo userInfo = convert(user);
+UserInfo userInfo = convert(user);
store.deleteResource(UserService.getId(userInfo.getUsername()));
store.putResource(UserService.getId(userInfo.getUsername()), userInfo, 0,
UserService.UserInfoSerializer.getInstance());
result = rs.next();
@@ -252,10 +252,10 @@ public class AclTableMigrationTool {
return newInfo;
}
-private UserService.UserInfo convert(User user) {
+private UserInfo convert(User user) {
if (user == null)
return null;
-UserService.UserInfo newInfo = new UserService.UserInfo();
+UserInfo newInfo = new UserInfo();
newInfo.setUsername(user.getUserName());
newInfo.setPassword(user.getPassword());
List authorities = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c18526c/server-base/src/main/java/org/apache/kylin/rest/service/UserGrantedAuthority.java
--
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/UserGrantedAuthority.java
b/server-base/src/main/java/org/apache/kylin/rest/service/UserGrantedAuthority.java
new file mode 100644
index 000..4c2a392
--- /dev/null
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/UserGrantedAuthority.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import org.springframework.security.core.GrantedAuthority;
+
+
+public class UserGrantedAuthority implements GrantedAuthority {
+private static final long serialVersionUID = -5128905636841891058L;
+
+private String authority;
+
+public UserGrantedAuthority() {
+}
+
+public UserGrantedAuthority(String authority) {
+setAuthority(authority);
+}
+
+@Override
+public String getAuthority() {
+return authority;
+}
+
+public void setAuthority(String authority) {
+this.authority = authority;
+}
+
+@Override
+public int hashCode() {
+final int prime = 31;
+int result = 1;
+result = prime * result + ((authority == null) ? 0 :
authority.hashCode());
+return result;
+}
+
+@
[31/37] kylin git commit: KYLIN-2627 add simple rollback on ResourceStore
KYLIN-2627 add simple rollback on ResourceStore
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea5cabac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea5cabac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea5cabac
Branch: refs/heads/master
Commit: ea5cabac95b3d27aa60d6b16263904142e3d2daa
Parents: c9dc7cc
Author: Li Yang
Authored: Wed May 17 17:46:08 2017 +0800
Committer: Roger Shi
Committed: Wed May 17 19:40:34 2017 +0800
--
.../kylin/common/persistence/ResourceStore.java | 103 ++-
.../persistence/LocalFileResourceStoreTest.java | 44
2 files changed, 145 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea5cabac/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index d5fbc2e..0565c66 100644
---
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -18,7 +18,9 @@
package org.apache.kylin.common.persistence;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
@@ -27,6 +29,7 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.NavigableSet;
import java.util.UUID;
@@ -237,6 +240,11 @@ abstract public class ResourceStore {
final public void putResource(String resPath, InputStream content, long
ts) throws IOException {
resPath = norm(resPath);
logger.trace("Directly saving resource " + resPath + " (Store " +
kylinConfig.getMetadataUrl() + ")");
+putResourceCheckpoint(resPath, content, ts);
+}
+
+private void putResourceCheckpoint(String resPath, InputStream content,
long ts) throws IOException {
+beforeChange(resPath);
putResourceImpl(resPath, content, ts);
}
@@ -266,7 +274,7 @@ abstract public class ResourceStore {
dout.close();
buf.close();
-newTS = checkAndPutResourceImpl(resPath, buf.toByteArray(), oldTS,
newTS);
+newTS = checkAndPutResourceCheckpoint(resPath, buf.toByteArray(),
oldTS, newTS);
obj.setLastModified(newTS); // update again the confirmed TS
return newTS;
} catch (IOException e) {
@@ -278,6 +286,11 @@ abstract public class ResourceStore {
}
}
+private long checkAndPutResourceCheckpoint(String resPath, byte[] content,
long oldTS, long newTS) throws IOException {
+beforeChange(resPath);
+return checkAndPutResourceImpl(resPath, content, oldTS, newTS);
+}
+
/**
* checks old timestamp when overwriting existing
*/
@@ -288,7 +301,12 @@ abstract public class ResourceStore {
*/
final public void deleteResource(String resPath) throws IOException {
logger.trace("Deleting resource " + resPath + " (Store " +
kylinConfig.getMetadataUrl() + ")");
-deleteResourceImpl(norm(resPath));
+deleteResourceCheckpoint(norm(resPath));
+}
+
+private void deleteResourceCheckpoint(String resPath) throws IOException {
+beforeChange(resPath);
+deleteResourceImpl(resPath);
}
abstract protected void deleteResourceImpl(String resPath) throws
IOException;
@@ -315,6 +333,87 @@ abstract public class ResourceStore {
//
+ThreadLocal checkpointing = new ThreadLocal<>();
+
+public Checkpoint checkpoint() {
+Checkpoint cp = checkpointing.get();
+if (cp != null)
+throw new IllegalStateException("A checkpoint has been open for
this thread: " + cp);
+
+cp = new Checkpoint();
+checkpointing.set(cp);
+return cp;
+}
+
+private void beforeChange(String resPath) throws IOException {
+Checkpoint cp = checkpointing.get();
+if (cp != null)
+cp.beforeChange(resPath);
+}
+
+public class Checkpoint implements Closeable {
+
+LinkedHashMap origResData = new LinkedHashMap<>();
+LinkedHashMap origResTimestamp = new LinkedHashMap<>();
+
+private void beforeChange(String resPath) throws IOException {
+if (origResData.containsKey(resPath))
+return;
+
+Raw
[30/37] kylin git commit: KYLIN-2625 not null filter clause should be evaluable in storage
KYLIN-2625 not null filter clause should be evaluable in storage
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c9dc7cc7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c9dc7cc7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c9dc7cc7
Branch: refs/heads/master
Commit: c9dc7cc78c5eed40f4c5a438423b857a145f17b1
Parents: a38b02d
Author: Hongbin Ma
Authored: Tue May 16 18:04:03 2017 +0800
Committer: Roger Shi
Committed: Tue May 16 20:59:12 2017 +0800
--
.../metadata/filter/CompareTupleFilter.java | 3 ++-
.../test/resources/query/sql_limit/query04.sql | 25
2 files changed, 27 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9dc7cc7/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
--
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index d783e52..2771250 100755
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -215,7 +215,8 @@ public class CompareTupleFilter extends TupleFilter {
@Override
public boolean isEvaluable() {
return (column != null || (function != null &&
function.isEvaluable())) //
-&& !conditionValues.isEmpty() && secondColumn == null;
+&& (!conditionValues.isEmpty() || operator ==
FilterOperatorEnum.ISNOTNULL || operator == FilterOperatorEnum.ISNULL) //
+&& secondColumn == null;
}
public boolean alwaysReturnTrue() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9dc7cc7/kylin-it/src/test/resources/query/sql_limit/query04.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_limit/query04.sql
b/kylin-it/src/test/resources/query/sql_limit/query04.sql
new file mode 100644
index 000..381ab65
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_limit/query04.sql
@@ -0,0 +1,25 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+
+select * from (
+select * from test_kylin_fact
+ where lstg_format_name is not null
+ ) limit 20
+
[21/37] kylin git commit: KYLIN-2597 Deal with trivial expression in filters like x = 1+2
KYLIN-2597 Deal with trivial expression in filters like x = 1+2
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/529b1be2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/529b1be2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/529b1be2
Branch: refs/heads/master
Commit: 529b1be2f9f0f1c7bfb2798928b8c6c60cb581ea
Parents: 9a812d5
Author: Hongbin Ma
Authored: Tue May 9 17:30:03 2017 +0800
Committer: Dong Li
Committed: Tue May 9 20:30:21 2017 +0800
--
.../test/resources/query/sql_limit/query03.sql | 31 +++
.../kylin/query/relnode/OLAPFilterRel.java | 55
2 files changed, 86 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/529b1be2/kylin-it/src/test/resources/query/sql_limit/query03.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_limit/query03.sql
b/kylin-it/src/test/resources/query/sql_limit/query03.sql
new file mode 100644
index 000..4844395
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_limit/query03.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ test_kylin_fact.lstg_format_name ,test_kylin_fact.cal_dt ,
sum(test_kylin_fact.price) as GMV, count(*) as TRANS_CNT
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND
test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ where test_kylin_fact.seller_id = 1002 + 4
+
+ group by seller_id,test_kylin_fact.cal_dt,lstg_format_name
+ limit 10
http://git-wip-us.apache.org/repos/asf/kylin/blob/529b1be2/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 0ee9204..06ad0c6 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.relnode;
+import java.math.BigDecimal;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.LinkedList;
@@ -68,6 +69,7 @@ import org.apache.kylin.metadata.filter.function.Functions;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -140,6 +142,16 @@ public class OLAPFilterRel extends Filter implements
OLAPRel {
case OTHER_FUNCTION:
filter = Functions.getFunctionTupleFilter(op.getName());
break;
+case PLUS:
+case MINUS:
+case TIMES:
+case DIVIDE:
+TupleFilter f = dealWithTrivialExpr(call);
+if (f != null) {
+// is a trivial expr
+return f;
+}
+//else go to default
default:
filter = new
UnsupportedTupleFilter(FilterOperatorEnum.UNSUPPORTED);
break;
@@ -166,6 +178,49 @@ public class OLAPFilterRel extends Filter implements
OLAPRel {
return filter;
}
+//KYLIN-2597 - Deal with trivial expression in filters like x = 1 + 2
+private TupleFilter dealWithTrivialExpr(RexCall call) {
+ImmutableList operators = call.operands;
+if (operators.size() != 2) {
+return null;
+}
+
+BigDecimal left = null;
+BigDecimal right = null;
+
[07/37] kylin git commit: change mvn compile to mvn package
change mvn compile to mvn package Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c58b983 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c58b983 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c58b983 Branch: refs/heads/master Commit: 4c58b9837bfff6493051a7f0a4a68132e59b9ef2 Parents: 2b6a91f Author: Hongbin Ma Authored: Wed May 3 11:37:41 2017 +0800 Committer: Dong Li Committed: Wed May 3 11:40:19 2017 +0800 -- dev-support/sync_hbase_cdh_branches.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/4c58b983/dev-support/sync_hbase_cdh_branches.sh -- diff --git a/dev-support/sync_hbase_cdh_branches.sh b/dev-support/sync_hbase_cdh_branches.sh index 592053a..340c570 100755 --- a/dev-support/sync_hbase_cdh_branches.sh +++ b/dev-support/sync_hbase_cdh_branches.sh @@ -45,7 +45,7 @@ git checkout -b tmp git reset apache/$base --hard git am -3 --ignore-whitespace 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch -mvn clean compile -DskipTests +mvn clean package -DskipTests git push apache tmp:$base-hbase0.98 -f rm 0001-KYLIN-2307-Create-a-branch-for-master-with-HBase-0.9.patch
[20/37] kylin git commit: KYLIN-2598 Should not translate filter to a in-clause filter with too many elements
KYLIN-2598 Should not translate filter to a in-clause filter with too many
elements
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9dc6553d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9dc6553d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9dc6553d
Branch: refs/heads/master
Commit: 9dc6553d29eb3f106823026f45dc878347a2a108
Parents: 529b1be
Author: Hongbin Ma
Authored: Tue May 9 18:33:21 2017 +0800
Committer: Dong Li
Committed: Tue May 9 20:30:21 2017 +0800
--
.../apache/kylin/common/KylinConfigBase.java| 4 +++
.../kylin/dict/BuiltInFunctionTransformer.java | 10 +-
.../test/resources/query/sql_like/query01.sql | 6 ++--
.../test/resources/query/sql_like/query05.sql | 33
.../test/resources/query/sql_like/query06.sql | 33
.../test/resources/query/sql_like/query22.sql | 31 ++
.../test/resources/query/sql_like/query23.sql | 31 ++
7 files changed, 144 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index eafdbbb..10209d7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -844,6 +844,10 @@ abstract public class KylinConfigBase implements
Serializable {
return value > 0 ? value : Long.MAX_VALUE;
}
+public int getTranslatedInClauseMaxSize() {
+return
Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size",
"1024"));
+}
+
public int getLargeQueryThreshold() {
return
Integer.parseInt(getOptional("kylin.query.large-query-threshold",
String.valueOf(100)));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
--
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 6ef2f96..4d7b15d 100755
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict;
import java.util.Collection;
import java.util.ListIterator;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dimension.IDimensionEncodingMap;
import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
@@ -93,12 +94,19 @@ public class BuiltInFunctionTransformer implements
ITupleFilterTransformer {
translated.addChild(new ColumnTupleFilter(columnRef));
try {
+int translatedInClauseMaxSize =
KylinConfig.getInstanceFromEnv().getTranslatedInClauseMaxSize();
+
for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
Object dictVal = dict.getValueFromId(i);
if ((Boolean)
builtInFunctionTupleFilter.invokeFunction(dictVal)) {
translated.addChild(new ConstantTupleFilter(dictVal));
+
+if (translated.getChildren().size() >
translatedInClauseMaxSize) {
+return null;
+}
}
}
+
} catch (Exception e) {
logger.debug(e.getMessage());
return null;
@@ -175,4 +183,4 @@ public class BuiltInFunctionTransformer implements
ITupleFilterTransformer {
}
return translated;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9dc6553d/kylin-it/src/test/resources/query/sql_like/query01.sql
--
diff --git a/kylin-it/src/test/resources/query/sql_like/query01.sql
b/kylin-it/src/test/resources/query/sql_like/query01.sql
index e1f131d..e3a05ca 100644
--- a/kylin-it/src/test/resources/query/sql_like/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_like/query01.sql
@@ -16,7 +16,7 @@
-- limitations under the License.
--
-select lstg_format_name as lstg_format_name, count(*) as cnt
+select META_CATEG_NAME as META_CATEG_NAME, count(*) as cnt
from test_kylin_fact
inner JOIN edw.test_cal_dt as test_cal_dt
@@ -27,5 +27,5 @@ inner JOIN edw.test_cal
[33/37] kylin git commit: KYLIN-2577, refine hive configurations during cubing steps
KYLIN-2577, refine hive configurations during cubing steps
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c152c51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c152c51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c152c51
Branch: refs/heads/master
Commit: 7c152c515ce6cebe380c4e1b7c16d109192308db
Parents: 0061900
Author: Cheng Wang
Authored: Thu May 18 11:05:34 2017 +0800
Committer: liyang-gmt8
Committed: Thu May 18 15:27:36 2017 +0800
--
build/bin/load-hive-conf.sh | 2 +-
.../kylin/common/util/HiveCmdBuilder.java | 56 +++-
.../kylin/common/util/HiveCmdBuilderTest.java | 2 +-
.../kylin/job/engine/JobEngineConfig.java | 43 ---
.../source/hive/CreateFlatHiveTableStep.java| 3 --
.../apache/kylin/source/hive/HiveMRInput.java | 4 --
6 files changed, 57 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/build/bin/load-hive-conf.sh
--
diff --git a/build/bin/load-hive-conf.sh b/build/bin/load-hive-conf.sh
index a5046f0..a42cd0c 100644
--- a/build/bin/load-hive-conf.sh
+++ b/build/bin/load-hive-conf.sh
@@ -11,5 +11,5 @@ hive_conf_prop="${KYLIN_HOME}/logs/hive_props"
rm -rf ${hive_conf_prop}
export ENABLE_CHECK_ENV=false
${dir}/kylin.sh org.apache.kylin.tool.HiveConfigCLI ${hive_conf_dir}
${hive_conf_prop}
-[[ 0 == $? ]] || quit "Error, can not parse ${hive_conf_dir} and can not apply
it to hive relevant check."
+[[ 0 == $? ]] || quit "Can not parse xml file: ${hive_conf_dir}, please check
it."
hive_conf_properties=`cat ${hive_conf_prop}`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
--
diff --git
a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 255867a..2f6b9a0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -27,17 +27,25 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
import com.google.common.collect.Lists;
public class HiveCmdBuilder {
private static final Logger logger =
LoggerFactory.getLogger(HiveCmdBuilder.class);
+public static final String HIVE_CONF_FILENAME = "kylin_hive_conf";
+
public enum HiveClientMode {
CLI, BEELINE
}
@@ -50,6 +58,7 @@ public class HiveCmdBuilder {
public HiveCmdBuilder() {
kylinConfig = KylinConfig.getInstanceFromEnv();
clientMode =
HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase());
+loadHiveConfiguration();
}
public String build() {
@@ -75,8 +84,8 @@ public class HiveCmdBuilder {
bw.newLine();
}
buf.append("beeline ");
-buf.append(parseProps());
buf.append(kylinConfig.getHiveBeelineParams());
+buf.append(parseProps());
buf.append(" -f ");
buf.append(tmpHql.getAbsolutePath());
buf.append(";ret_code=$?;rm -f ");
@@ -123,6 +132,7 @@ public class HiveCmdBuilder {
}
public void setHiveConfProps(Map hiveConfProps) {
+this.hiveConfProps.clear();
this.hiveConfProps.putAll(hiveConfProps);
}
@@ -144,4 +154,48 @@ public class HiveCmdBuilder {
public String toString() {
return build();
}
+
+private void loadHiveConfiguration() {
+
+File hiveConfFile;
+String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml");
+String path = System.getProperty(KylinConfig.KYLIN_CONF);
+
+if (StringUtils.isNotEmpty(path)) {
+hiveConfFile = new File(path, hiveConfFileName);
+} else {
+path = KylinConfig.getKylinHome();
+if (StringUtils.isEmpty(path)) {
+logger.error("KYLIN_HOME is not set, can not locate hive conf:
{}.xml", HIVE_CONF_FILENAME);
+return;
+}
+hiveConfFile = new File(path + File.separator + "conf",
hiveConfFileN
[05/37] kylin git commit: KYLIN-2471 support subquery for parenthesized sub-clause in JOIN
KYLIN-2471 support subquery for parenthesized sub-clause in JOIN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0b51f7be
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0b51f7be
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0b51f7be
Branch: refs/heads/master
Commit: 0b51f7bec9d6f721d8e6a092f23b2f1d445f7a18
Parents: 299b584
Author: Dong Li
Authored: Mon May 1 11:35:07 2017 +0800
Committer: hongbin ma
Committed: Mon May 1 11:35:07 2017 +0800
--
.../query/util/CognosParenthesesEscape.java | 114 +++
.../query/util/CognosParentesesEscapeTest.java | 61 --
.../query/util/CognosParenthesesEscapeTest.java | 81 +
.../src/test/resources/query/cognos/query02.sql | 27 +
.../resources/query/cognos/query02.sql.expected | 27 +
.../src/test/resources/query/cognos/query03.sql | 29 +
.../resources/query/cognos/query03.sql.expected | 29 +
7 files changed, 262 insertions(+), 106 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kylin/blob/0b51f7be/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
--
diff --git
a/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
b/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
index 70ab8da..6d930a5 100644
---
a/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
+++
b/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
@@ -6,77 +6,101 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
package org.apache.kylin.query.util;
-import java.util.LinkedList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
-import java.util.UUID;
+import java.util.Map;
+import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.kylin.common.util.Pair;
+import com.google.common.collect.Lists;
-/**
- * from (a join b on a.x = b.y) join c
- *
- * similar in https://issues.apache.org/jira/browse/CALCITE-35
- *
- * we'll find such pattern and remove the parentheses
- */
public class CognosParenthesesEscape implements QueryUtil.IQueryTransformer {
-
-private static final String S0 = "\\s*";
-private static final String S1 = "\\s";
-private static final String SM = "\\s+";
-private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+";
-private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS =
TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?";
-private static final String JOIN =
"(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per
http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server
-private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME +
S0 + "=" + S0 + TABLE_OR_COLUMN_NAME;
-private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // (
-TABLE_NAME_WITH_OPTIONAL_ALIAS + // a
-JOIN + SM + // join
-TABLE_NAME_WITH_OPTIONAL_ALIAS + //b
-SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*"
+ // on a.x = b.y [and a.x2 = b.y2]
-S0 + "\\)";// )
-private static final Pattern PARENTTHESES_PATTERN =
Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE);
-
-private static int identifierNum = 0;
+private static final Pattern FROM_PATTERN =
Pattern.compile("\\s+from\\s+(\\s*\\(\\s*)+(?!\\s*select\\s)",
Pattern.CASE_INSENSITIVE);
@Override
public String transform(String sql) {
+if (sql == null || sql.isEmpty()) {
+return sql;
+}
+
+Map parenthesesPairs = findParenthesesPairs(sql);
+if (parenthesesPairs.isEmpty()) {
+// parentheses not found
+return sql;
+}
+
+List parentheses = Lists.newArrayList();
+StringBuilder result = new StringBuilder(sql);
+
Matcher m;
-List> matches = new LinkedList<>();
while (true) {
-m = PARENTTHESES_PATTERN.matcher(sql);
-if (!m.find())
+m = FROM_PATTERN.matcher(sql);
+i
