add installation guide
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/56ba5131 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/56ba5131 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/56ba5131 Branch: refs/heads/inverted-index Commit: 56ba5131a780ce3ada62c9068efab49b67e679b1 Parents: 443b192 Author: qianhao.zhou <qianz...@ebay.com> Authored: Thu Feb 12 10:35:46 2015 +0800 Committer: qianhao.zhou <qianz...@ebay.com> Committed: Thu Feb 12 10:35:46 2015 +0800 ---------------------------------------------------------------------- bin/health-check.sh | 33 ++++++++++ bin/healthmon.sh | 33 ---------- .../org/apache/kylin/common/KylinConfig.java | 3 +- conf/kylin.properties | 8 +-- docs/Installation/install_and_run.md | 69 ++++++++++++++++++++ .../test_case_data/localmeta/kylin.properties | 6 -- .../localmeta_v1/kylin.properties | 6 -- .../minicluster/b-kylin/kylin.properties | 6 -- .../minicluster/b-kylin/meta/kylin.properties | 6 -- .../test_case_data/minicluster/kylin.properties | 6 -- .../sandbox-hdp21/kylin.properties | 6 -- .../test_case_data/sandbox/kylin.properties | 6 -- server/ServiceConfig.md | 4 +- 13 files changed, 105 insertions(+), 87 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/bin/health-check.sh ---------------------------------------------------------------------- diff --git a/bin/health-check.sh b/bin/health-check.sh new file mode 100644 index 0000000..439eef4 --- /dev/null +++ b/bin/health-check.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +ALERT="y...@email.com" + +OUTPUT=$( + curl --max-time 20 -# \ + --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \ + -H "Authorization:Basic QURNSU46S1lMSU4=" \ + -H "Content-Type:application/json;charset=UTF-8" \ + http://localhost:7070/kylin/api/query \ +) + +# ---------------------------------------------------------------------------- + +date + +if [[ $OUTPUT == *"results"* ]]; then + echo "Good." +else + echo "Bad." + TS_FILE=/tmp/kylin_healthmon_ts + LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null` + CURR_TS=`date +%s` + echo last: $LAST_TS + echo curr: $CURR_TS + if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then + echo "Sending mail..." + echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT + if [ "$?" == "0" ]; then + touch $TS_FILE + fi + fi +fi http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/bin/healthmon.sh ---------------------------------------------------------------------- diff --git a/bin/healthmon.sh b/bin/healthmon.sh deleted file mode 100644 index 439eef4..0000000 --- a/bin/healthmon.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -ALERT="y...@email.com" - -OUTPUT=$( - curl --max-time 20 -# \ - --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \ - -H "Authorization:Basic QURNSU46S1lMSU4=" \ - -H "Content-Type:application/json;charset=UTF-8" \ - http://localhost:7070/kylin/api/query \ -) - -# ---------------------------------------------------------------------------- - -date - -if [[ $OUTPUT == *"results"* ]]; then - echo "Good." -else - echo "Bad." - TS_FILE=/tmp/kylin_healthmon_ts - LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null` - CURR_TS=`date +%s` - echo last: $LAST_TS - echo curr: $CURR_TS - if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then - echo "Sending mail..." - echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT - if [ "$?" == "0" ]; then - touch $TS_FILE - fi - fi -fi http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java index ef2c6e2..add87e0 100644 --- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -89,9 +89,8 @@ public class KylinConfig { public static final String KYLIN_JOB_JAR = "kylin.job.jar"; public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar"; - public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold"; - public static final String KYLIN_JOB_JAR_LOCAL = "kylin.job.jar.local"; + public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold"; public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir"; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/conf/kylin.properties b/conf/kylin.properties index bf86a26..f607dca 100755 --- a/conf/kylin.properties +++ b/conf/kylin.properties @@ -4,20 +4,14 @@ kylin.rest.servers= # The metadata store in hbase -kylin.metadata.url=kylin_metadata_qa@hbase +kylin.metadata.url=kylin_metadata@hbase # The storage for final cube file in hbase kylin.storage.url=hbase -# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase -kylin.coprocessor.local.jar= - # Temp folder in hdfs kylin.hdfs.working.dir=/tmp -# Path to the local(relative to job engine) job jar, job engine will use this jar -kylin.job.jar= - kylin.job.mapreduce.default.reduce.input.mb=500 # If true, job engine will not assume that hadoop CLI reside on the same server as it self http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/docs/Installation/install_and_run.md ---------------------------------------------------------------------- diff --git a/docs/Installation/install_and_run.md b/docs/Installation/install_and_run.md new file mode 100644 index 0000000..d634b32 --- /dev/null +++ b/docs/Installation/install_and_run.md @@ -0,0 +1,69 @@ +##Install and Run + +###How to run + +1. Download the release version(according to the hadoop distribution) +2. Setup a KYLIN_HOME pointing to the corresponding directory where you extract the release tar +3. Make sure the user has the privilege to run hadoop, hive and hbase cmd in shell. If you are not so sure, you can just run **bin/check-env.sh**, it will print out the detail information if you have some environment issues. +4. To start Kylin, simply run **bin/start-kylin.sh** +5. To stop Kylin, simply run **bin/stop-kylin.sh** + + +If you are running Kylin in a cluster or you have multiple Kylin instances, please make sure you have the following property correctly configured. + +1. kylin.rest.servers + + List of web servers in use, this enables one web server instance to sync up with other servers. + + +2. kylin.server.mode + + Make sure there is only one instance whose "kylin.server.mode" is set to "all" if there are multiple instances. + + +###The directory structure of the installation + +> âââ bin +> â âââ check-env.sh +> â âââ find-hive-dependency.sh +> â âââ health-check.sh +> â âââ start-kylin.sh +> â âââ stop-kylin.sh +> â +> âââ conf +> â âââ kylin_job_conf.xml +> â âââ kylin.properties +> â +> âââ tomcat +> â âââ webapps +> â â âââ kylin.war +> â â¦â¦. +> â +> âââ lib +> âââ kylin-coprocessor-${version}.jar +> âââ kylin-job-${version}.jar + + + +###Build from source +> git clone https://github.com/KylinOLAP/Kylin.git +> cd KylinOLAP/Kylin +> sh script/package.sh + +In order to generate binary package, **maven** and **npm** are pre-requisites. + + + + + + + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/localmeta/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index 53d24ee..aa18287 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -9,15 +9,9 @@ kylin.metadata.url= # The storage for final cube file in hbase kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure -# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase -kylin.coprocessor.local.jar=/tmp/kylin/kylin-coprocessor-latest.jar - # Temp folder in hdfs kylin.hdfs.working.dir=/tmp -# Path to the local(relative to job engine) job jar, job engine will use this jar -kylin.job.jar=/tmp/kylin/kylin-job-latest.jar - kylin.job.mapreduce.default.reduce.input.mb=500 # If true, job engine will not assume that hadoop CLI reside on the same server as it self http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/localmeta_v1/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta_v1/kylin.properties b/examples/test_case_data/localmeta_v1/kylin.properties index 96439b4..aa18287 100644 --- a/examples/test_case_data/localmeta_v1/kylin.properties +++ b/examples/test_case_data/localmeta_v1/kylin.properties @@ -9,15 +9,9 @@ kylin.metadata.url= # The storage for final cube file in hbase kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure -# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase -kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar - # Temp folder in hdfs kylin.hdfs.working.dir=/tmp -# Path to the local(relative to job engine) job jar, job engine will use this jar -kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar - kylin.job.mapreduce.default.reduce.input.mb=500 # If true, job engine will not assume that hadoop CLI reside on the same server as it self http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/minicluster/b-kylin/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/minicluster/b-kylin/kylin.properties b/examples/test_case_data/minicluster/b-kylin/kylin.properties index 19c8de6..5d1b667 100644 --- a/examples/test_case_data/minicluster/b-kylin/kylin.properties +++ b/examples/test_case_data/minicluster/b-kylin/kylin.properties @@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u # The storage for final cube file in hbase kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure -# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase -kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar - # Temp folder in hdfs kylin.hdfs.working.dir=/tmp -# Path to the local(relative to job engine) job jar, job engine will use this jar -kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar - kylin.job.mapreduce.default.reduce.input.mb=500 # If true, job engine will not assume that hadoop CLI reside on the same server as it self http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties index 419b9e8..17e5c99 100644 --- a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties +++ b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties @@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u # The storage for final cube file in hbase kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure -# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase -kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar - # Temp folder in hdfs kylin.hdfs.working.dir=/tmp -# Path to the local(relative to job engine) job jar, job engine will use this jar -kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar - kylin.job.mapreduce.default.reduce.input.mb=500 # If true, job engine will not assume that hadoop CLI reside on the same server as it self http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/minicluster/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties index 7816cad..373186c 100644 --- a/examples/test_case_data/minicluster/kylin.properties +++ b/examples/test_case_data/minicluster/kylin.properties @@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:minicluster:/hbase-unsecure # The storage for final cube file in hbase kylin.storage.url=hbase:minicluster:/hbase-unsecure -# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase -kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar - # Temp folder in hdfs kylin.hdfs.working.dir=/tmp -# Path to the local(relative to job engine) job jar, job engine will use this jar -kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar - kylin.job.mapreduce.default.reduce.input.mb=500 # If true, job engine will not assume that hadoop CLI reside on the same server as it self http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/sandbox-hdp21/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox-hdp21/kylin.properties b/examples/test_case_data/sandbox-hdp21/kylin.properties index 7474283..3e2f97a 100755 --- a/examples/test_case_data/sandbox-hdp21/kylin.properties +++ b/examples/test_case_data/sandbox-hdp21/kylin.properties @@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u # The storage for final cube file in hbase kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure -# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase -kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar - # Temp folder in hdfs kylin.hdfs.working.dir=/tmp -# Path to the local(relative to job engine) job jar, job engine will use this jar -kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar - kylin.job.mapreduce.default.reduce.input.mb=500 # If true, job engine will not assume that hadoop CLI reside on the same server as it self http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index adc0dda..cd5d02c 100755 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase # The storage for final cube file in hbase kylin.storage.url=hbase -# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase -kylin.coprocessor.local.jar= - # Temp folder in hdfs kylin.hdfs.working.dir=/tmp -# Path to the local(relative to job engine) job jar, job engine will use this jar -kylin.job.jar= - kylin.job.mapreduce.default.reduce.input.mb=500 # If true, job engine will not assume that hadoop CLI reside on the same server as it self http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/server/ServiceConfig.md ---------------------------------------------------------------------- diff --git a/server/ServiceConfig.md b/server/ServiceConfig.md index 0f951be..fdeba33 100644 --- a/server/ServiceConfig.md +++ b/server/ServiceConfig.md @@ -20,8 +20,6 @@ ##### kylin.query.cache.enabled ### Job -##### kylin.job.jar -##### kylin.job.log.dir ##### kylin.job.hdfs.working.dir ##### kylin.job.mapreduce.default.reduce.input.mb ##### kylin.job.mapreduce.default.reduce.count.ratio @@ -35,6 +33,6 @@ ##### kylin.job.concurrent.max.limit ##### kylin.job.command.exe.type ##### kylin.job.admin.dls -##### kylin.job.step.timeout +##### kylin.job.step.timeoutww ##### kylin.job.yarn.app.rest.check.interval.seconds ##### kylin.job.concurrent.max.limit \ No newline at end of file
