Author: mahongbin
Date: Wed Feb 3 08:26:52 2016
New Revision: 1728267
URL: http://svn.apache.org/viewvc?rev=1728267&view=rev
Log:
UPDATE MESSAGE
Added:
kylin/site/development/streaming_cubing.html
kylin/site/images/develop/streaming.png (with props)
Modified:
kylin/site/development/about_temp_files.html
kylin/site/development/dev_env.html
kylin/site/development/howto_contribute.html
kylin/site/development/howto_docs.html
kylin/site/development/howto_package.html
kylin/site/development/howto_release.html
kylin/site/development/howto_test.html
kylin/site/development/index.html
kylin/site/development/new_metadata.html
kylin/site/development/web_tech.html
kylin/site/feed.xml
Modified: kylin/site/development/about_temp_files.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/about_temp_files.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/about_temp_files.html (original)
+++ kylin/site/development/about_temp_files.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
Modified: kylin/site/development/dev_env.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/dev_env.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/dev_env.html (original)
+++ kylin/site/development/dev_env.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_contribute.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/howto_contribute.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_contribute.html (original)
+++ kylin/site/development/howto_contribute.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_docs.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/howto_docs.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_docs.html (original)
+++ kylin/site/development/howto_docs.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_package.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/howto_package.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_package.html (original)
+++ kylin/site/development/howto_package.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_release.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/howto_release.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_release.html (original)
+++ kylin/site/development/howto_release.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
Modified: kylin/site/development/howto_test.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/howto_test.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/howto_test.html (original)
+++ kylin/site/development/howto_test.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
@@ -441,7 +476,7 @@
<article
class="post-content" >
<p>In general, there
should be unit tests to cover individual classes; there must be integration
test to cover end-to-end scenarios like build, merge, and query. Unit test must
run independently (does not require an external sandbox).</p>
-<h2 id="x-branches">2.x branches</h2>
+<h2 id="test-2x-branches">Test 2.x branches</h2>
<ul>
<li><code class="highlighter-rouge">mvn test</code> to run unit tests, which
has a limited test coverage.
@@ -453,15 +488,15 @@
</li>
<li><code
class="highlighter-rouge">dev-support/test_all_against_hdp_2_2_4_2_2.sh</code>
to run integration tests, which has the best test coverage.
<ul>
- <li>Integration tests <strong>must run on a Hadoop sandbox</strong>.
Make sure all changes you want to test are avaiable on sandbox.</li>
+ <li>Integration tests <strong>better be run on a Hadoop
sandbox</strong>. We suggest to checkout a copy of code in your sandbox and
direct run the test_all_against_hdp_2_2_4_2_2.sh in it. If you donât want to
put codes on sandbox, refer to <strong>More on 2.x UT/IT
separation</strong></li>
<li>As the name indicates, the script is only for hdp 2.2.4.2, but you
get the idea of how integration test run from it.</li>
<li>The integration tests start from generate random data, then build
cube, merge cube, and finally query the result and compare to H2 DB.</li>
- <li>The integration tests take a few hours to complete.</li>
+ <li>The integration tests take one to two hours to complete.</li>
</ul>
</li>
</ul>
-<h2 id="x-branches-1">1.x branches</h2>
+<h2 id="test-1x-branches">Test 1.x branches</h2>
<ul>
<li><code class="highlighter-rouge">mvn test</code> to run unit tests, which
has a limited test coverage.
@@ -480,6 +515,32 @@
</li>
</ul>
+<h2 id="more-on-2x-utit-separation">More on 2.x UT/IT separation</h2>
+
+<p>From Kylin 2.0 you can run UT(Unit test), environment cube provision and
IT(Integration test) separately. <br />
+Running <code class="highlighter-rouge">mvn verify
-Dhdp.version=2.2.4.2-2</code> (assume youâre on your sandbox) is all you
need to run a complete all the test suites.</p>
+
+<p>It will execute the following steps sequentially:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>1. Build Artifacts
+2. Run all UTs (takes few minutes)
+3. Provision cubes on the sandbox environment for IT uasge (takes 1~2 hours)
+4. Run all ITs (takes few tens of minutes)
+5. verify jar stuff
+</code></pre>
+</div>
+
+<p>If your code change is minor and it merely requires running UT, use: <br />
+<code class="highlighter-rouge">mvn test</code><br />
+If your sandbox is already provisioned and your code change will not affect
the result of sandbox provision, (and you donât want to wait hours of
provision) just run the following commands to separately run UT and IT: <br />
+<code class="highlighter-rouge">mvn test</code><br />
+<code class="highlighter-rouge">mvn failsafe:integration-test</code></p>
+
+<h3 id="cube-provision">Cube Provision</h3>
+
+<p>Environment cube provision is indeed running kylin cubing jobs to prepare
example cubes in the sandbox. These prepared cubes will be used by the ITs.
Currently provision step is bound with the maven pre-integration-test phase,
and it contains running BuildCubeWithEngine (HBase required),
BuildCubeWithStream(Kafka required) and BuildIIWithStream(Kafka Required). You
can run the mvn commands on you sandbox or your develop computer. For the
latter case you need to set kylin.job.run.as.remote.cmd=true in
<strong>$KYLIN_HOME/examples/test_case_data/sandbox/kylin.properties</strong>.
<br />
+Try appending <code class="highlighter-rouge">-DfastBuildMode=true</code> to
mvn verify command to speed up provision by skipping incremental cubing.</p>
+
<h2 id="more-on-1x-mini-cluster">More on 1.x Mini Cluster</h2>
<p>Kylin 1.x used to move as many as possible unit test cases from sandbox to
HBase mini cluster (not any more in 2.x), so that user can run tests easily in
local without a hadoop sandbox. Two maven profiles are created in the root
pom.xml, âdefaultâ and âsandboxâ. The default profile will startup a
HBase Mini Cluster to prepare the test data and run the unit tests (the test
cases that are not supported by Mini cluster have been added in the
âexcludeâ list). If you want to keep using Sandbox to run test, just run
<code class="highlighter-rouge">mvn test -P sandbox</code></p>
@@ -494,6 +555,7 @@
</ul>
<h3 id="to-ensure-mini-cluster-can-run-successfully-you-need">To ensure Mini
cluster can run successfully, you need</h3>
+
<ul>
<li>Make sure JAVA_HOME is properly set</li>
</ul>
Modified: kylin/site/development/index.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/index.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/index.html (original)
+++ kylin/site/development/index.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
Modified: kylin/site/development/new_metadata.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/new_metadata.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/new_metadata.html (original)
+++ kylin/site/development/new_metadata.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
Added: kylin/site/development/streaming_cubing.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/streaming_cubing.html?rev=1728267&view=auto
==============================================================================
--- kylin/site/development/streaming_cubing.html (added)
+++ kylin/site/development/streaming_cubing.html Wed Feb 3 08:26:52 2016
@@ -0,0 +1,551 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<!doctype html>
+<html>
+ <!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+
+ <title>Apache Kylin | Streaming cubing (Prototype)</title>
+ <meta name="description" content="Apache Kylin Home">
+ <meta name="author" content="Apache Kylin">
+ <link rel="shortcut icon" href="fav.png" type="image/png">
+
+
+
+<link rel="stylesheet" href="/assets/css/animate.css">
+<!-- Bootstrap -->
+<link rel="stylesheet" href="/assets/css/bootstrap.min.css">
+
+<!-- Fonts -->
+<!-- <link rel="stylesheet"
href="http://fonts.googleapis.com/css?family=Alice|Open+Sans:400,300,700"> -->
+
+<!-- Icons -->
+<link rel="stylesheet" href="/assets/css/font-awesome.min.css">
+
+ <!-- Custom styles -->
+ <link rel="stylesheet" href="/assets/css/styles.css">
+ <link rel="stylesheet" href="/assets/css/docs.css">
+ <link rel="stylesheet" href="/assets/css/pygments.css">
+
+ <link rel="canonical"
href="http://kylin.apache.org/development/streaming_cubing.html";>
+ <link rel="alternate" type="application/rss+xml" title="Apache Kylin"
href="http://kylin.apache.org/feed.xml"; />
+
+<!--[if lt IE 9]> <script src="assets/js/html5shiv.js"></script> <![endif]-->
+<script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new
Date();a=s.createElement(o),
+
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ //oringal tracker for kylin.io
+ ga('create', 'UA-55534813-1', 'auto');
+ //new tracker for kylin.apache.org
+ ga('create', 'UA-55534813-2', 'auto', {'name':'toplevel'});
+
+ ga('send', 'pageview');
+ ga('toplevel.send', 'pageview');
+
+
+</script>
+<script type="text/javascript" src="/assets/js/jquery-1.9.1.min.js"></script>
+<script type="text/javascript" src="/assets/js/nside.js"></script> </script>
+<script type="text/javascript" src="/assets/js/nnav.js"></script> </script>
+</head>
+
+ <body>
+ <!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<header id="header" >
+
+ <div id="head" class="parallax" parallax-speed="3" >
+ <div id="logo" class="text-center"> <img class="img-circle"
id="circlelogo" src="/assets/images/kylin_logo.jpg"> <span class="title"
>Apache Kylinâ¢</span> <span class="tagline">Extreme OLAP Engine for Big
Data</span>
+ </div>
+ </div>
+
+
+ <!-- Main Menu -->
+ <nav class="navbar navbar-default" role="navigation" id="nav-wrapper">
+ <div class="container-fluid" id="nav">
+ <!--
+ <img class="img-circle" width="40px" height="40px" id="circlelogo"
src="/assets/images/kylin_logo.jpg">
+ -->
+ <!-- Brand and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed"
data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
+ </div>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="/">Home</a></li>
+ <li><a href="/docs" >Docs</a></li>
+ <li><a href="/download">Download</li>
+ <li><a href="/community" >Community</a></li>
+ <li><a href="/development" >Development</a></li>
+ <li><a href="/blog">Blog</li>
+ <li><a href="/cn" >ä¸æç</a></li>
+ <li><a href="https://twitter.com/apachekylin"; target="_blank"
class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" ></a></li>
+ <li><a href="https://github.com/apache/kylin"; target="_blank"
class="fa fa-github-alt fa-lg" title="Github: apache/kylin" ></a></li>
+ <li><a href="https://www.facebook.com/kylinio"; target="_blank"
class="fa fa-facebook fa-lg" title="Facebook: kylin.io" ></a></li>
+ </ul>
+ </div><!-- /.navbar-collapse -->
+ </div><!-- /.container-fluid -->
+</nav>
+ </header>
+
+
+ <div class="container">
+ <div class="row">
+ <!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<div class="col-md-3 col-lg-3 col-xs-4 aside1 visible-md visible-lg"
id="nside1" style=" padding-top: 2em">
+ <ul class="nav nav-pills nav-stacked">
+
+ <li><a href="#titleDevelopment Guide" data-toggle="collapse"
id="navtitle">Development Guide</a></li>
+ <div class="collapse in">
+ <div class="list-group" id="list1">
+ <ul style="list-style-type:disc">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/index.html" class="list-group-item-lay
pjaxlink" id="navlist">Development Quick Guide</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/dev_env.html" class="list-group-item-lay
pjaxlink" id="navlist">Setup Development Env</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_test.html" class="list-group-item-lay
pjaxlink" id="navlist">How to Test</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_contribute.html"
class="list-group-item-lay pjaxlink" id="navlist">How to Contribute</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_docs.html" class="list-group-item-lay
pjaxlink" id="navlist">How to Write Document</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_package.html" class="list-group-item-lay
pjaxlink" id="navlist">How to Build Binary Package</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/howto_release.html" class="list-group-item-lay
pjaxlink" id="navlist">How to Making a Release</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/new_metadata.html" class="list-group-item-lay
pjaxlink" id="navlist">New Metadata Model</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/about_temp_files.html"
class="list-group-item-lay pjaxlink" id="navlist">About Temp Files</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
+
+
+
+ <ul>
+ </div>
+</div>
+
+
+ </ul>
+</div>
+ <div class="col-md-9 col-lg-9 col-xs-14 aside2">
+ <div id="container">
+ <div id="pjax">
+ <h1
class="post-title">Streaming cubing (Prototype)</h1>
+ <article
class="post-content" >
+ <p>One of the most
important features in 2.x branches is streaming cubing which enables OLAP
analysis on streaming data. Streaming cubing delivers faster insights on the
data to help more promptly business decisions. Even though there are already
many real time analysis tools in open source community, Kylin Streaming cubing
still differs from them in multiple angles:</p>
+
+<p>Firstly, Kylin Streaming Cubing aligns with Kylin traditional cubing to
provided unified, ANSI SQL interface. Actually Kylin Streaming shares the
storage engine and query engine with traditional Kylin cubes, so in theory all
of the optimization techniques to save storage and speed up query performance
can also be applied on streaming cubes. Besides, all the supported
aggregations/filters/UDFs still work for streaming cubes. By unifying the
storage engine and query engine we also get freed from double amount of
maintaince work.</p>
+
+<p>Secondly, Kylin Streaming Cubing does not require large amount of memory to
store real time data, nor does it attempts to provide truly âreal timeâ
analysis. By our customer survey we found that minutes of visualization latency
is acceptable for OLAP analysts. So our streaming cubing adopts the micro batch
approach. Incoming streaming data are partitioned into different time windows
and we build a micro batch for each time window. The cube output for each micro
batch is directly saved to HBase. The query engine goes to HBase for data
retrieving instead of the data ingestion server. The benefit of such design is
that we donât have to maintain large amount of in-memory index which could
easily require tens of gigabytes of memory. In the future Kylin might need to
consider truly âreal timeâ support, too.</p>
+
+<p>Thirdly, Kylin Streaming Cubing data will be persistent and gradually be
converted to traditional cubes, thus customers can still query âcold dataâ
without any compromise on performance. As discussed above the output of
streaming cubing is directly saved to HBase as a new segment. The traditional
job engine will be notified of the new segment and take over to schedule merge
jobs when then segments accumulates. Day after day the segments of the
streaming cube got merged and become a very large traditional cube.</p>
+
+<p><img src="/images/develop/streaming.png" alt="Kylin Streaming Framework
Architecture" /></p>
+
+<p>With the major difference in mind we will introduce the modules for Kylin
Streaming cubing. Kylin Streaming cubing consist of three major parts:</p>
+
+<ul>
+ <li>Streaming Input to retrieve data from a replayable data queue (currently
it is Kafka) within given time window. Streaming Input is also responsible for
primary data cleaning and normalization. By default Kylin Streaming provides a
default implementation to parse the messages from the source queue. Customers
can choose to configure the parser or provide a brand new one based on their
requirements.</li>
+ <li>Streaming Batch Ingestion to ingest the incoming data batch and
transform it into a micro cube. Thanks to the latest Kylin In-memory cubing
technology, this step is now times faster and space-saving than previous. The
micro cube is directly saved to HBase.</li>
+ <li>Job Scheduling Module to trigger Streaming Batch Ingestion. Kylin does
not put too much efforts in job scheduling, streaming cubing is not a
exception. Currently we provided a simple implementation based on Linux
Crontab.</li>
+</ul>
+
+<p>Weâll publish more detailed documents on how to use Kylin Streaming soon.
In latest 2.x branch we are also working on more complicated load balancing
schemes for streaming cubing. Please stay tuned.</p>
+
+
+ </article>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<footer id="underfooter">
+ <div class="container">
+ <div class="row">
+ <div class="col-md-12 widget">
+ <div class="widget-body" style="text-align:center">
+ <a href="http://www.apache.org";>
+ <img id="asf-logo" alt="Apache Software Foundation"
src="/assets/images/feather-small.gif">
+ </a>
+
+ <div>
+ The contents of this website are © 2015 Apache
Software Foundation under the terms of the <a
+ href="http://www.apache.org/licenses/LICENSE-2.0";>
Apache License v2 </a>. Apache Kylin and
+ its logo are trademarks of the Apache Software
Foundation.
+ </div>
+
+ </div>
+ </div>
+ </div>
+ <!-- /row of widgets -->
+
+ </div>
+ <div></div>
+
+</footer>
+
+ <script src="/assets/js/jquery-1.9.1.min.js"></script>
+ <script src="/assets/js/bootstrap.min.js"></script>
+ <script src="/assets/js/main.js"></script>
+ </body>
+</html>
Modified: kylin/site/development/web_tech.html
URL:
http://svn.apache.org/viewvc/kylin/site/development/web_tech.html?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/development/web_tech.html (original)
+++ kylin/site/development/web_tech.html Wed Feb 3 08:26:52 2016
@@ -405,6 +405,8 @@
+
+
<li><a href="/development/web_tech.html" class="list-group-item-lay
pjaxlink" id="navlist">Kylin Web Summary</a></li>
@@ -424,6 +426,39 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/development/streaming_cubing.html"
class="list-group-item-lay pjaxlink" id="navlist">Streaming cubing
(Prototype)</a></li>
+
+
+
Modified: kylin/site/feed.xml
URL:
http://svn.apache.org/viewvc/kylin/site/feed.xml?rev=1728267&r1=1728266&r2=1728267&view=diff
==============================================================================
--- kylin/site/feed.xml (original)
+++ kylin/site/feed.xml Wed Feb 3 08:26:52 2016
@@ -19,8 +19,8 @@
<description>Apache Kylin Home</description>
<link>http://kylin.apache.org/</link>
<atom:link href="http://kylin.apache.org/feed.xml"; rel="self"
type="application/rss+xml"/>
- <pubDate>Mon, 01 Feb 2016 06:48:24 -0800</pubDate>
- <lastBuildDate>Mon, 01 Feb 2016 06:48:24 -0800</lastBuildDate>
+ <pubDate>Wed, 03 Feb 2016 00:25:14 -0800</pubDate>
+ <lastBuildDate>Wed, 03 Feb 2016 00:25:14 -0800</lastBuildDate>
<generator>Jekyll v2.5.3</generator>
<item>
Added: kylin/site/images/develop/streaming.png
URL:
http://svn.apache.org/viewvc/kylin/site/images/develop/streaming.png?rev=1728267&view=auto
==============================================================================
Binary file - no diff available.
Propchange: kylin/site/images/develop/streaming.png
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
