Repository: kylin
Updated Branches:
  refs/heads/master e95c07156 -> a60f893ff


KYLIN-2450 sample.sh generates sample data to 
kylin.source.hive.database-for-flat-table


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a60f893f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a60f893f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a60f893f

Branch: refs/heads/master
Commit: a60f893ff0c6d8f1e34ef9bbdd3cab7cc875aa99
Parents: e95c071
Author: Billy Liu <billy...@apache.org>
Authored: Thu Feb 16 17:36:49 2017 +0800
Committer: Billy Liu <billy...@apache.org>
Committed: Thu Feb 16 17:36:49 2017 +0800

----------------------------------------------------------------------
 build/bin/sample.sh                             | 16 +++++++++--
 build/conf/kylin.properties                     |  2 +-
 .../org/apache/kylin/common/util/Bytes.java     | 10 +++----
 .../kylin/common/util/CliCommandExecutor.java   |  1 +
 .../cube/inmemcubing/InMemCubeBuilderUtils.java |  2 +-
 .../kylin/engine/mr/common/CubeStatsReader.java |  2 +-
 .../kylin/engine/mr/common/NDCuboidBuilder.java |  2 +-
 .../steps/RowKeyDistributionCheckerMapper.java  |  2 +-
 examples/sample_cube/create_sample_tables.sql   | 30 ++++++++++----------
 .../apache/kylin/jdbc/json/SQLResponseStub.java |  2 +-
 .../apache/kylin/source/hive/SchemaChecker.java |  2 +-
 11 files changed, 41 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 1363a5d..74991c0 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -22,7 +22,6 @@ source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
 source ${dir}/check-env.sh "if-not-yet"
 job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
 
-echo "Going to create sample tables in hive..."
 cd ${KYLIN_HOME}/sample_cube/data
 
 echo "Loading sample data into HDFS tmp path: /tmp/kylin/sample_cube/data"
@@ -30,12 +29,17 @@ hadoop fs -mkdir -p /tmp/kylin/sample_cube/data
 hadoop fs -put * /tmp/kylin/sample_cube/data/
 
 hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh 
kylin.source.hive.client`
+sample_database=`bash ${KYLIN_HOME}/bin/get-properties.sh 
kylin.source.hive.database-for-flat-table`
+echo "Going to create sample tables in hive to database "$sample_database" by 
"$hive_client_mode
 if [ "${hive_client_mode}" == "beeline" ]
 then
     beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh 
kylin.source.hive.beeline-params`
-    beeline ${beeline_params} -f 
${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }
+    beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS 
"$sample_database
+#### Caution: hard code the beeline parameter by appending sample database.
+    beeline ${beeline_params}"/"$sample_database -f 
${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }
 else
-    hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }
+    hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database
+    hive --database ${sample_database} -f 
${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || { exit 1; }
 fi
 
 echo "Sample hive tables are created successfully; Going to create sample 
cube..."
@@ -54,6 +58,12 @@ cp -rf ${KYLIN_HOME}/sample_cube/template/* 
${KYLIN_HOME}/sample_cube/metadata
 
 sed -i "s/%default_storage_type%/${default_storage_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
 sed -i "s/%default_engine_type%/${default_engine_type}/g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/DEFAULT./${sample_database}./g" 
${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/DEFAULT./${sample_database}./g" 
${KYLIN_HOME}/sample_cube/metadata/model_desc/kylin_sales_model.json
+sed -i "s/DEFAULT./${sample_database}./g" 
${KYLIN_HOME}/sample_cube/metadata/project/learn_kylin.json
+
+cd ${KYLIN_HOME}/sample_cube/metadata/table
+ls -1 DEFAULT.KYLIN_*.json|sed "s/\(DEFAULT\)\(.*\)\.json/mv & 
${sample_database}\2.json/"|sh -v
 
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} 
org.apache.kylin.common.persistence.ResourceTool upload 
${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index f2ebb80..12c4c9c 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -47,7 +47,7 @@ kylin.web.cross-domain-enabled=true
 kylin.source.hive.client=cli
 
 # Parameters for beeline client, only necessary if hive client is beeline
-#kylin.source.hive.beeline-params=-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u 'jdbc:hive2://localhost:10000'
+#kylin.source.hive.beeline-params=-n root --hiveconf 
hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' 
-u jdbc:hive2://localhost:10000
 
 kylin.source.hive.keep-flat-table=false
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java 
b/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java
index f64f4b2..fc501ea 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java
@@ -328,7 +328,7 @@ public class Bytes {
                     continue;
                 }
                 // turn hex ASCII digit -> number
-                byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + 
toBinaryFromHex((byte) hd2));
+                byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + 
(toBinaryFromHex((byte) hd2)) & 0xff);
 
                 b[size++] = d;
                 i += 3; // skip 3
@@ -1118,8 +1118,8 @@ public class Bytes {
                 }
                 final int minLength = Math.min(length1, length2);
                 final int minWords = minLength / SIZEOF_LONG;
-                final long offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
-                final long offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
+                final long offset1Adj = offset1 + (long)BYTE_ARRAY_BASE_OFFSET;
+                final long offset2Adj = offset2 + (long)BYTE_ARRAY_BASE_OFFSET;
 
                 /*
                  * Compare 8 bytes at a time. Benchmarking shows comparing 8 
bytes at a
@@ -1455,7 +1455,7 @@ public class Bytes {
         if (inclusive) {
             diffBI = diffBI.add(BigInteger.ONE);
         }
-        final BigInteger splitsBI = BigInteger.valueOf(num + 1);
+        final BigInteger splitsBI = BigInteger.valueOf(num + 1L);
         if (diffBI.compareTo(splitsBI) < 0) {
             return null;
         }
@@ -1942,7 +1942,7 @@ public class Bytes {
         hex = hex.toUpperCase();
         byte[] b = new byte[hex.length() / 2];
         for (int i = 0; i < b.length; i++) {
-            b[i] = (byte) ((toBinaryFromHex((byte) hex.charAt(2 * i)) << 4) + 
toBinaryFromHex((byte) hex.charAt((2 * i + 1))));
+            b[i] = (byte) ((toBinaryFromHex((byte) hex.charAt(2 * i)) << 4) + 
(toBinaryFromHex((byte) hex.charAt((2 * i + 1))) & 0xff));
         }
         return b;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
 
b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index f97b609..9d41c86 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -142,6 +142,7 @@ public class CliCommandExecutor {
             int exitCode = proc.waitFor();
             return Pair.newPair(exitCode, result.toString());
         } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
             throw new IOException(e);
         }
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
index a0b4d46..8557acf 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -41,7 +41,7 @@ public final class InMemCubeBuilderUtils {
         ImmutableBitSet measureColumns = 
parentDimensionAndMetricColumnBitSet.getSecond();
         ImmutableBitSet childDimensions = parentDimensions;
         long mask = Long.highestOneBit(baseCuboidId);
-        long parentCuboidIdActualLength = Long.SIZE - 
Long.numberOfLeadingZeros(baseCuboidId);
+        long parentCuboidIdActualLength = (long)Long.SIZE - 
Long.numberOfLeadingZeros(baseCuboidId);
         int index = 0;
         for (int i = 0; i < parentCuboidIdActualLength; i++) {
             if ((mask & baseCuboidId) > 0) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index dbe4554..9cb60f8 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -201,7 +201,7 @@ public class CubeStatsReader {
         KylinConfig kylinConf = cubeSegment.getConfig();
 
         long mask = Long.highestOneBit(baseCuboidId);
-        long parentCuboidIdActualLength = Long.SIZE - 
Long.numberOfLeadingZeros(baseCuboidId);
+        long parentCuboidIdActualLength = (long)Long.SIZE - 
Long.numberOfLeadingZeros(baseCuboidId);
         for (int i = 0; i < parentCuboidIdActualLength; i++) {
             if ((mask & cuboidId) > 0) {
                 rowkeyLength += rowKeyColumnLength.get(i); 
//colIO.getColumnLength(columnList.get(i));

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
----------------------------------------------------------------------
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
index 4e98618..9ab42ea 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
@@ -67,7 +67,7 @@ public class NDCuboidBuilder implements Serializable {
         long mask = Long.highestOneBit(parentCuboid.getId());
         long parentCuboidId = parentCuboid.getId();
         long childCuboidId = childCuboid.getId();
-        long parentCuboidIdActualLength = Long.SIZE - 
Long.numberOfLeadingZeros(parentCuboid.getId());
+        long parentCuboidIdActualLength = (long)Long.SIZE - 
Long.numberOfLeadingZeros(parentCuboid.getId());
         int index = rowKeySplitter.getBodySplitOffset(); // skip shard and 
cuboidId
         for (int i = 0; i < parentCuboidIdActualLength; i++) {
             if ((mask & parentCuboidId) > 0) {// if the this bit position 
equals

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
----------------------------------------------------------------------
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
index a2efe04..eab57d1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
@@ -68,7 +68,7 @@ public class RowKeyDistributionCheckerMapper extends 
KylinMapper<Text, Text, Tex
         for (Text t : keyList) {
             if (key.compareTo(t) < 0) {
                 Long v = resultMap.get(t);
-                long length = key.getLength() + value.getLength();
+                long length = (long)key.getLength() + value.getLength();
                 v += length;
                 resultMap.put(t, v);
                 break;

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql 
b/examples/sample_cube/create_sample_tables.sql
index bdfc4d7..24f26f8 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -16,9 +16,9 @@
 -- limitations under the License.
 --
 
-DROP TABLE IF EXISTS DEFAULT.KYLIN_CAL_DT;
+DROP TABLE IF EXISTS KYLIN_CAL_DT;
 
-CREATE TABLE DEFAULT.KYLIN_CAL_DT
+CREATE TABLE KYLIN_CAL_DT
 (
 CAL_DT date COMMENT 'Date, PK'
 ,YEAR_BEG_DT date COMMENT 'YEAR Begin Date'
@@ -125,9 +125,9 @@ COMMENT 'Date Dimension Table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
-DROP TABLE IF EXISTS DEFAULT.KYLIN_CATEGORY_GROUPINGS;
+DROP TABLE IF EXISTS KYLIN_CATEGORY_GROUPINGS;
 
-CREATE TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS
+CREATE TABLE KYLIN_CATEGORY_GROUPINGS
 (
 LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
 ,LEAF_CATEG_NAME string
@@ -170,9 +170,9 @@ COMMENT 'Detail category inforamtion, Dimension Table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
-DROP TABLE IF EXISTS DEFAULT.KYLIN_COUNTRY;
+DROP TABLE IF EXISTS KYLIN_COUNTRY;
 
-CREATE TABLE DEFAULT.KYLIN_COUNTRY
+CREATE TABLE KYLIN_COUNTRY
 (
 COUNTRY string
 ,LATITUDE double
@@ -182,9 +182,9 @@ COUNTRY string
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
-DROP TABLE IF EXISTS DEFAULT.KYLIN_ACCOUNT;
+DROP TABLE IF EXISTS KYLIN_ACCOUNT;
 
-CREATE TABLE DEFAULT.KYLIN_ACCOUNT
+CREATE TABLE KYLIN_ACCOUNT
 (
 ACCOUNT_ID bigint
 ,ACCOUNT_BUYER_LEVEL int COMMENT 'Account Buyer Level'
@@ -195,9 +195,9 @@ ACCOUNT_ID bigint
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
-DROP TABLE IF EXISTS DEFAULT.KYLIN_SALES;
+DROP TABLE IF EXISTS KYLIN_SALES;
 
-CREATE TABLE DEFAULT.KYLIN_SALES
+CREATE TABLE KYLIN_SALES
 (
 TRANS_ID bigint
 ,PART_DT date COMMENT 'Order Date'
@@ -216,8 +216,8 @@ COMMENT 'Sales order table, fact table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' 
OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' 
OVERWRITE INTO TABLE DEFAULT.KYLIN_ACCOUNT;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' 
OVERWRITE INTO TABLE DEFAULT.KYLIN_COUNTRY;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' 
OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT;
-LOAD DATA INPATH 
'/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE 
INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' 
OVERWRITE INTO TABLE KYLIN_SALES;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' 
OVERWRITE INTO TABLE KYLIN_ACCOUNT;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' 
OVERWRITE INTO TABLE KYLIN_COUNTRY;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' 
OVERWRITE INTO TABLE KYLIN_CAL_DT;
+LOAD DATA INPATH 
'/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE 
INTO TABLE KYLIN_CATEGORY_GROUPINGS;

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java 
b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
index a57177c..a05b6d6 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
@@ -154,7 +154,7 @@ public class SQLResponseStub implements Serializable {
     }
 
     @JsonIgnoreProperties(ignoreUnknown = true)
-    public static class ColumnMetaStub {
+    public static class ColumnMetaStub implements Serializable{
 
         private boolean isAutoIncrement;
         private boolean isCaseSensitive;

http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
----------------------------------------------------------------------
diff --git 
a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java 
b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index e2a94fb..3488fbd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -81,7 +81,7 @@ public class SchemaChecker {
                 buf.append("- ").append(reason).append("\n");
             }
 
-            return new CheckResult(false, format("Found %d issue(s) with 
'%s':\n%s Please disable and purge related cube(s) first", reasons.size(), 
tableName, buf.toString()));
+            return new CheckResult(false, format("Found %d issue(s) with 
'%s':%n%s Please disable and purge related cube(s) first", reasons.size(), 
tableName, buf.toString()));
         }
     }
 

Reply via email to