http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/content/troubleshooting.html ---------------------------------------------------------------------- diff --git a/content/troubleshooting.html b/content/troubleshooting.html deleted file mode 100644 index 7e0cd60..0000000 --- a/content/troubleshooting.html +++ /dev/null @@ -1,423 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - <title>CarbonData</title> - <style> - - </style> - <!-- Bootstrap --> - - <link rel="stylesheet" href="css/bootstrap.min.css"> - <link href="css/style.css" rel="stylesheet"> - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js";></script> - <![endif]--> - <script src="js/jquery.min.js"></script> - <script src="js/bootstrap.min.js"></script> - <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js";></script> - - -</head> -<body> -<header> - <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper"> - <div class="container"> - <div class="navbar-header"> - <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse" - class="navbar-toggle collapsed" type="button"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a href="index.html" class="logo"> - <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/> - </a> - </div> - <div class="navbar-collapse collapse cd_navcontnt" id="navbar"> - <ul class="nav navbar-nav navbar-right navlist-custom"> - <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a> - </li> - <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true" - aria-expanded="false"> Download <span class="caret"></span></a> - <ul class="dropdown-menu"> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; - target="_blank">Apache CarbonData 1.4.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; - target="_blank">Apache CarbonData 1.4.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"; - target="_blank">Apache CarbonData 1.3.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; - target="_blank">Apache CarbonData 1.3.0</a></li> - <li> - <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; - target="_blank">Release Archive</a></li> - </ul> - </li> - <li><a href="documentation.html" class="active">Documentation</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" - aria-expanded="false">Community <span class="caret"></span></a> - <ul class="dropdown-menu"> - <li> - <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"; - target="_blank">Contributing to CarbonData</a></li> - <li> - <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"; - target="_blank">Release Guide</a></li> - <li> - <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"; - target="_blank">Project PMC and Committers</a></li> - <li> - <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"; - target="_blank">CarbonData Meetups</a></li> - <li><a href="security.html">Apache CarbonData Security</a></li> - <li><a href="https://issues.apache.org/jira/browse/CARBONDATA"; target="_blank">Apache - Jira</a></li> - <li><a href="videogallery.html">CarbonData Videos </a></li> - </ul> - </li> - <li class="dropdown"> - <a href="http://www.apache.org/"; class="apache_link hidden-xs dropdown-toggle" - data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> - <ul class="dropdown-menu"> - <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> - <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> - <li><a href="http://www.apache.org/foundation/sponsorship.html"; - target="_blank">Sponsorship</a></li> - <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> - </ul> - </li> - - <li class="dropdown"> - <a href="http://www.apache.org/"; class="hidden-lg hidden-md hidden-sm dropdown-toggle" - data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> - <ul class="dropdown-menu"> - <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> - <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> - <li><a href="http://www.apache.org/foundation/sponsorship.html"; - target="_blank">Sponsorship</a></li> - <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> - </ul> - </li> - - <li> - <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a> - - </li> - - </ul> - </div><!--/.nav-collapse --> - <div id="search-box"> - <form method="get" action="http://www.google.com/search"; target="_blank"> - <div class="search-block"> - <table border="0" cellpadding="0" width="100%"> - <tr> - <td style="width:80%"> - <input type="text" name="q" size=" 5" maxlength="255" value="" - class="search-input" placeholder="Search...." required/> - </td> - <td style="width:20%"> - <input type="submit" value="Search"/></td> - </tr> - <tr> - <td align="left" style="font-size:75%" colspan="2"> - <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/> - <span style=" position: relative; top: -3px;"> Only search for CarbonData</span> - </td> - </tr> - </table> - </div> - </form> - </div> - </div> - </nav> -</header> <!-- end Header part --> - -<div class="fixed-padding"></div> <!-- top padding with fixde header --> - -<section><!-- Dashboard nav --> - <div class="container-fluid q"> - <div class="col-sm-12 col-md-12 maindashboard"> - <div class="verticalnavbar"> - <nav class="b-sticky-nav"> - <div class="nav-scroller"> - <div class="nav__inner"> - <a class="b-nav__intro nav__item" href="./videogallery.html">introduction</a> - <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> - <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a> - - <div class="nav__item nav__item__with__subs"> - <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> - <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a> - <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a> - <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a> - <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a> - <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a> - <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a> - </div> - - <div class="nav__item nav__item__with__subs"> - <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a> - <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a> - <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a> - <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a> - <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> - </div> - - <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> - <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> - <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> - <a class="b-nav__trouble nav__item" href="./troubleshooting.html">Troubleshooting</a> - <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> - <a class="b-nav__contri nav__item" href="./videogallery.html">Contribute</a> - <a class="b-nav__security nav__item" href="./security.html">Security</a> - <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a> - </div> - </div> - <div class="navindicator"> - <div class="b-nav__intro navindicator__item"></div> - <div class="b-nav__quickstart navindicator__item"></div> - <div class="b-nav__uses navindicator__item"></div> - <div class="b-nav__docs navindicator__item"></div> - <div class="b-nav__datamap navindicator__item"></div> - <div class="b-nav__s3 navindicator__item"></div> - <div class="b-nav__api navindicator__item"></div> - <div class="b-nav__perf navindicator__item"></div> - <div class="b-nav__trouble navindicator__item"></div> - <div class="b-nav__faq navindicator__item"></div> - <div class="b-nav__contri navindicator__item"></div> - <div class="b-nav__security navindicator__item"></div> - </div> - </nav> - </div> - <div class="mdcontent"> - <section> - <div style="padding:10px 15px;"> - <div id="viewpage" name="viewpage"> - <div class="row"> - <div class="col-sm-12 col-md-12"> - <div> -<h1> -<a id="troubleshooting" class="anchor" href="#troubleshooting" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Troubleshooting</h1> -<p>This tutorial is designed to provide troubleshooting for end users and developers -who are building, deploying, and using CarbonData.</p> -<h2> -<a id="when-loading-data-gets-tablestatuslock-issues" class="anchor" href="#when-loading-data-gets-tablestatuslock-issues" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>When loading data, gets tablestatus.lock issues:</h2> -<p><strong>Symptom</strong></p> -<pre><code>17/11/11 16:48:13 ERROR LocalFileLock: main hdfs:/localhost:9000/carbon/store/default/hdfstable/tablestatus.lock (No such file or directory) -java.io.FileNotFoundException: hdfs:/localhost:9000/carbon/store/default/hdfstable/tablestatus.lock (No such file or directory) - at java.io.FileOutputStream.open0(Native Method) - at java.io.FileOutputStream.open(FileOutputStream.java:270) - at java.io.FileOutputStream.<init>(FileOutputStream.java:213) - at java.io.FileOutputStream.<init>(FileOutputStream.java:101) -</code></pre> -<p><strong>Possible Cause</strong> -If you use <code><hdfs path></code> as store path when creating carbonsession, may get the errors,because the default is LOCALLOCK.</p> -<p><strong>Procedure</strong> -Before creating carbonsession, sets as below:</p> -<pre><code>import org.apache.carbondata.core.util.CarbonProperties -import org.apache.carbondata.core.constants.CarbonCommonConstants -CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOCK_TYPE, "HDFSLOCK") -</code></pre> -<h2> -<a id="failed-to-load-thrift-libraries" class="anchor" href="#failed-to-load-thrift-libraries" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to load thrift libraries</h2> -<p><strong>Symptom</strong></p> -<p>Thrift throws following exception :</p> -<pre><code>thrift: error while loading shared libraries: -libthriftc.so.0: cannot open shared object file: No such file or directory -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>The complete path to the directory containing the libraries is not configured correctly.</p> -<p><strong>Procedure</strong></p> -<p>Follow the Apache thrift docs at <a href="https://thrift.apache.org/docs/install"; target=_blank rel="nofollow">https://thrift.apache.org/docs/install</a> to install thrift correctly.</p> -<h2> -<a id="failed-to-launch-the-spark-shell" class="anchor" href="#failed-to-launch-the-spark-shell" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to launch the Spark Shell</h2> -<p><strong>Symptom</strong></p> -<p>The shell prompts the following error :</p> -<pre><code>org.apache.spark.sql.CarbonContext$$anon$$apache$spark$sql$catalyst$analysis -$OverrideCatalog$_setter_$org$apache$spark$sql$catalyst$analysis -$OverrideCatalog$$overrides_$e -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>The Spark Version and the selected Spark Profile do not match.</p> -<p><strong>Procedure</strong></p> -<ol> -<li> -<p>Ensure your spark version and selected profile for spark are correct.</p> -</li> -<li> -<p>Use the following command :</p> -</li> -</ol> -<pre><code>"mvn -Pspark-2.1 -Dspark.version {yourSparkVersion} clean package" -</code></pre> -<p>Note : Refrain from using "mvn clean package" without specifying the profile.</p> -<h2> -<a id="failed-to-execute-load-query-on-cluster" class="anchor" href="#failed-to-execute-load-query-on-cluster" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to execute load query on cluster.</h2> -<p><strong>Symptom</strong></p> -<p>Load query failed with the following exception:</p> -<pre><code>Dictionary file is locked for updation. -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>The carbon.properties file is not identical in all the nodes of the cluster.</p> -<p><strong>Procedure</strong></p> -<p>Follow the steps to ensure the carbon.properties file is consistent across all the nodes:</p> -<ol> -<li> -<p>Copy the carbon.properties file from the master node to all the other nodes in the cluster. -For example, you can use ssh to copy this file to all the nodes.</p> -</li> -<li> -<p>For the changes to take effect, restart the Spark cluster.</p> -</li> -</ol> -<h2> -<a id="failed-to-execute-insert-query-on-cluster" class="anchor" href="#failed-to-execute-insert-query-on-cluster" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to execute insert query on cluster.</h2> -<p><strong>Symptom</strong></p> -<p>Load query failed with the following exception:</p> -<pre><code>Dictionary file is locked for updation. -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>The carbon.properties file is not identical in all the nodes of the cluster.</p> -<p><strong>Procedure</strong></p> -<p>Follow the steps to ensure the carbon.properties file is consistent across all the nodes:</p> -<ol> -<li> -<p>Copy the carbon.properties file from the master node to all the other nodes in the cluster. -For example, you can use scp to copy this file to all the nodes.</p> -</li> -<li> -<p>For the changes to take effect, restart the Spark cluster.</p> -</li> -</ol> -<h2> -<a id="failed-to-connect-to-hiveuser-with-thrift" class="anchor" href="#failed-to-connect-to-hiveuser-with-thrift" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to connect to hiveuser with thrift</h2> -<p><strong>Symptom</strong></p> -<p>We get the following exception :</p> -<pre><code>Cannot connect to hiveuser. -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>The external process does not have permission to access.</p> -<p><strong>Procedure</strong></p> -<p>Ensure that the Hiveuser in mysql must allow its access to the external processes.</p> -<h2> -<a id="failed-to-read-the-metastore-db-during-table-creation" class="anchor" href="#failed-to-read-the-metastore-db-during-table-creation" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to read the metastore db during table creation.</h2> -<p><strong>Symptom</strong></p> -<p>We get the following exception on trying to connect :</p> -<pre><code>Cannot read the metastore db -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>The metastore db is dysfunctional.</p> -<p><strong>Procedure</strong></p> -<p>Remove the metastore db from the carbon.metastore in the Spark Directory.</p> -<h2> -<a id="failed-to-load-data-on-the-cluster" class="anchor" href="#failed-to-load-data-on-the-cluster" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to load data on the cluster</h2> -<p><strong>Symptom</strong></p> -<p>Data loading fails with the following exception :</p> -<pre><code>Data Load failure exception -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>The following issue can cause the failure :</p> -<ol> -<li> -<p>The core-site.xml, hive-site.xml, yarn-site and carbon.properties are not consistent across all nodes of the cluster.</p> -</li> -<li> -<p>Path to hdfs ddl is not configured correctly in the carbon.properties.</p> -</li> -</ol> -<p><strong>Procedure</strong></p> -<p>Follow the steps to ensure the following configuration files are consistent across all the nodes:</p> -<ol> -<li> -<p>Copy the core-site.xml, hive-site.xml, yarn-site,carbon.properties files from the master node to all the other nodes in the cluster. -For example, you can use scp to copy this file to all the nodes.</p> -<p>Note : Set the path to hdfs ddl in carbon.properties in the master node.</p> -</li> -<li> -<p>For the changes to take effect, restart the Spark cluster.</p> -</li> -</ol> -<h2> -<a id="failed-to-insert-data-on-the-cluster" class="anchor" href="#failed-to-insert-data-on-the-cluster" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to insert data on the cluster</h2> -<p><strong>Symptom</strong></p> -<p>Insertion fails with the following exception :</p> -<pre><code>Data Load failure exception -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>The following issue can cause the failure :</p> -<ol> -<li> -<p>The core-site.xml, hive-site.xml, yarn-site and carbon.properties are not consistent across all nodes of the cluster.</p> -</li> -<li> -<p>Path to hdfs ddl is not configured correctly in the carbon.properties.</p> -</li> -</ol> -<p><strong>Procedure</strong></p> -<p>Follow the steps to ensure the following configuration files are consistent across all the nodes:</p> -<ol> -<li> -<p>Copy the core-site.xml, hive-site.xml, yarn-site,carbon.properties files from the master node to all the other nodes in the cluster. -For example, you can use scp to copy this file to all the nodes.</p> -<p>Note : Set the path to hdfs ddl in carbon.properties in the master node.</p> -</li> -<li> -<p>For the changes to take effect, restart the Spark cluster.</p> -</li> -</ol> -<h2> -<a id="failed-to-execute-concurrent-operationsloadinsertupdate-on-table-by-multiple-workers" class="anchor" href="#failed-to-execute-concurrent-operationsloadinsertupdate-on-table-by-multiple-workers" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to execute Concurrent Operations(Load,Insert,Update) on table by multiple workers.</h2> -<p><strong>Symptom</strong></p> -<p>Execution fails with the following exception :</p> -<pre><code>Table is locked for updation. -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>Concurrency not supported.</p> -<p><strong>Procedure</strong></p> -<p>Worker must wait for the query execution to complete and the table to release the lock for another query execution to succeed.</p> -<h2> -<a id="failed-to-create-a-table-with-a-single-numeric-column" class="anchor" href="#failed-to-create-a-table-with-a-single-numeric-column" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Failed to create a table with a single numeric column.</h2> -<p><strong>Symptom</strong></p> -<p>Execution fails with the following exception :</p> -<pre><code>Table creation fails. -</code></pre> -<p><strong>Possible Cause</strong></p> -<p>Behaviour not supported.</p> -<p><strong>Procedure</strong></p> -<p>A single column that can be considered as dimension is mandatory for table creation.</p> -<script> -// Show selected style on nav item -$(function() { $('.b-nav__trouble').addClass('selected'); }); -</script> -</div> -</div> -</div> -</div> -<div class="doc-footer"> - <a href="#top" class="scroll-top">Top</a> -</div> -</div> -</section> -</div> -</div> -</div> -</section><!-- End systemblock part --> -<script src="js/custom.js"></script> -</body> -</html>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/content/usecases.html ---------------------------------------------------------------------- diff --git a/content/usecases.html b/content/usecases.html new file mode 100644 index 0000000..cb309dd --- /dev/null +++ b/content/usecases.html @@ -0,0 +1,619 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>CarbonData</title> + <style> + + </style> + <!-- Bootstrap --> + + <link rel="stylesheet" href="css/bootstrap.min.css"> + <link href="css/style.css" rel="stylesheet"> + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js";></script> + <![endif]--> + <script src="js/jquery.min.js"></script> + <script src="js/bootstrap.min.js"></script> + <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js";></script> + + +</head> +<body> +<header> + <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper"> + <div class="container"> + <div class="navbar-header"> + <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse" + class="navbar-toggle collapsed" type="button"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a href="index.html" class="logo"> + <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/> + </a> + </div> + <div class="navbar-collapse collapse cd_navcontnt" id="navbar"> + <ul class="nav navbar-nav navbar-right navlist-custom"> + <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a> + </li> + <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false"> Download <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; + target="_blank">Apache CarbonData 1.4.0</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"; + target="_blank">Apache CarbonData 1.3.1</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; + target="_blank">Release Archive</a></li> + </ul> + </li> + <li><a href="documentation.html" class="active">Documentation</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false">Community <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"; + target="_blank">Contributing to CarbonData</a></li> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"; + target="_blank">Release Guide</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"; + target="_blank">Project PMC and Committers</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"; + target="_blank">CarbonData Meetups</a></li> + <li><a href="security.html">Apache CarbonData Security</a></li> + <li><a href="https://issues.apache.org/jira/browse/CARBONDATA"; target="_blank">Apache + Jira</a></li> + <li><a href="videogallery.html">CarbonData Videos </a></li> + </ul> + </li> + <li class="dropdown"> + <a href="http://www.apache.org/"; class="apache_link hidden-xs dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li class="dropdown"> + <a href="http://www.apache.org/"; class="hidden-lg hidden-md hidden-sm dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li> + <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a> + + </li> + + </ul> + </div><!--/.nav-collapse --> + <div id="search-box"> + <form method="get" action="http://www.google.com/search"; target="_blank"> + <div class="search-block"> + <table border="0" cellpadding="0" width="100%"> + <tr> + <td style="width:80%"> + <input type="text" name="q" size=" 5" maxlength="255" value="" + class="search-input" placeholder="Search...." required/> + </td> + <td style="width:20%"> + <input type="submit" value="Search"/></td> + </tr> + <tr> + <td align="left" style="font-size:75%" colspan="2"> + <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/> + <span style=" position: relative; top: -3px;"> Only search for CarbonData</span> + </td> + </tr> + </table> + </div> + </form> + </div> + </div> + </nav> +</header> <!-- end Header part --> + +<div class="fixed-padding"></div> <!-- top padding with fixde header --> + +<section><!-- Dashboard nav --> + <div class="container-fluid q"> + <div class="col-sm-12 col-md-12 maindashboard"> + <div class="verticalnavbar"> + <nav class="b-sticky-nav"> + <div class="nav-scroller"> + <div class="nav__inner"> + <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> + <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> + <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> + <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a> + <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a> + <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a> + <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a> + <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a> + <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a> + </div> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a> + <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a> + <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a> + <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a> + <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> + </div> + + <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> + <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a> + <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> + <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> + <a class="b-nav__security nav__item" href="./security.html">Security</a> + <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a> + </div> + </div> + <div class="navindicator"> + <div class="b-nav__intro navindicator__item"></div> + <div class="b-nav__quickstart navindicator__item"></div> + <div class="b-nav__uses navindicator__item"></div> + <div class="b-nav__docs navindicator__item"></div> + <div class="b-nav__datamap navindicator__item"></div> + <div class="b-nav__api navindicator__item"></div> + <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> + <div class="b-nav__faq navindicator__item"></div> + <div class="b-nav__contri navindicator__item"></div> + <div class="b-nav__security navindicator__item"></div> + </div> + </nav> + </div> + <div class="mdcontent"> + <section> + <div style="padding:10px 15px;"> + <div id="viewpage" name="viewpage"> + <div class="row"> + <div class="col-sm-12 col-md-12"> + <div><h1> +<a id="use-cases" class="anchor" href="#use-cases" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Use Cases</h1> +<p>CarbonData is useful in various analytical work loads.Some of the most typical usecases where CarbonData is being used is documented here.</p> +<p>CarbonData is used for but not limited to</p> +<ul> +<li> +<h3> +<a id="bank" class="anchor" href="#bank" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Bank</h3> +<ul> +<li>fraud detection analysis</li> +<li>risk profile analysis</li> +<li>As a zip table to update the daily balance of customers</li> +</ul> +</li> +<li> +<h3> +<a id="telecom" class="anchor" href="#telecom" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Telecom</h3> +<ul> +<li>Detection of signal anamolies for VIP customers for providing improved customer experience</li> +<li>Analysis of MR,CHR records of GSM data to determine the tower load at a particular time period and rebalance the tower configuration</li> +<li>Analysis of access sites, video, screen size, streaming bandwidth, quality to determine the network quality,routing configuration</li> +</ul> +</li> +<li> +<h3> +<a id="webinternet" class="anchor" href="#webinternet" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Web/Internet</h3> +<ul> +<li>Analysis of page or video being accessed,server loads, streaming quality, screen size</li> +</ul> +</li> +<li> +<h3> +<a id="smart-city" class="anchor" href="#smart-city" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Smart City</h3> +<ul> +<li>Vehicle tracking analysis</li> +<li>Unusual behaviour analysis</li> +</ul> +</li> +</ul> +<p>These use cases can be broadly classified into below categories:</p> +<ul> +<li>Full scan/Detailed/Interactive queries</li> +<li>Aggregation/OLAP BI queries</li> +<li>Real time Ingestion(Streaming) and queries</li> +</ul> +<h2> +<a id="detailed-queries-in-the-telecom-scenario" class="anchor" href="#detailed-queries-in-the-telecom-scenario" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Detailed Queries in the Telecom scenario</h2> +<h3> +<a id="scenario" class="anchor" href="#scenario" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Scenario</h3> +<p>User wants to analyse all the CHR(Call History Record) and MR(Measurement Records) of the mobile subscribers in order to identify the service failures within 10 secs.Also user wants to run machine learning models on the data to fairly estimate the reasons and time of probable failures and take action ahead to meet the SLA(Service Level Agreements) of VIP customers.</p> +<h3> +<a id="challenges" class="anchor" href="#challenges" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Challenges</h3> +<ul> +<li>Data incoming rate might vary based on the user concentration at a particular period of time.Hence higher data load speeds are required</li> +<li>Cluster needs to be well utilised and share the cluster among various applications for better resource consumption and savings</li> +<li>Queries needs to be interactive.ie., the queries fetch small data and need to be returned in seconds</li> +<li>Data Loaded into the system every few minutes.</li> +</ul> +<h3> +<a id="solution" class="anchor" href="#solution" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Solution</h3> +<p>Setup a Hadoop + Spark + CarbonData cluster managed by YARN.</p> +<p>Proposed the following configurations for CarbonData.(These tunings were proposed before CarbonData introduced SORT_COLUMNS parameter using which the sort order and schema order could be different.)</p> +<p>Add the frequently used columns to the left of the table definition.Add it in the increasing order of cardinality.It was suggested to keep msisdn,imsi columns in the beginning of the schema.With latest CarbonData, SORT_COLUMNS needs to be configured msisdn,imsi in the beginning.</p> +<p>Add timestamp column to the right of the schema as it is naturally increasing.</p> +<p>Create two separate YARN queues for Query and Data Loading.</p> +<p>Apart from these, the following CarbonData configuration was suggested to be configured in the cluster.</p> +<table> +<thead> +<tr> +<th>Configuration for</th> +<th>Parameter</th> +<th>Value</th> +<th>Description</th> +</tr> +</thead> +<tbody> +<tr> +<td>Data Loading</td> +<td>carbon.graph.rowset.size</td> +<td>100000</td> +<td>Based on the size of each row, this determines the memory required during data loading.Higher value leads to increased memory foot print</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.number.of.cores.while.loading</td> +<td>12</td> +<td>More cores can improve data loading speed</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.sort.size</td> +<td>100000</td> +<td>Number of records to sort at a time.More number of records configured will lead to increased memory foot print</td> +</tr> +<tr> +<td>Data Loading</td> +<td>table_blocksize</td> +<td>256</td> +<td>To efficiently schedule multiple tasks during query</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.sort.intermediate.files.limit</td> +<td>100</td> +<td>Increased to 100 as number of cores are more.Can perform merging in backgorund.If less number of files to merge, sort threads would be idle</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.use.local.dir</td> +<td>TRUE</td> +<td>yarn application directory will be usually on a single disk.YARN would be configured with multiple disks to be used as temp or to assign randomly to applications.Using the yarn temp directory will allow carbon to use multiple disks and improve IO performance</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.use.multiple.temp.dir</td> +<td>TRUE</td> +<td>multiple disks to write sort files will lead to better IO and reduce the IO bottleneck</td> +</tr> +<tr> +<td>Compaction</td> +<td>carbon.compaction.level.threshold</td> +<td>6,6</td> +<td>Since frequent small loads, compacting more segments will give better query results</td> +</tr> +<tr> +<td>Compaction</td> +<td>carbon.enable.auto.load.merge</td> +<td>true</td> +<td>Since data loading is small,auto compacting keeps the number of segments less and also compaction can complete in time</td> +</tr> +<tr> +<td>Compaction</td> +<td>carbon.number.of.cores.while.compacting</td> +<td>4</td> +<td>Higher number of cores can improve the compaction speed</td> +</tr> +<tr> +<td>Compaction</td> +<td>carbon.major.compaction.size</td> +<td>921600</td> +<td>Sum of several loads to combine into single segment</td> +</tr> +</tbody> +</table> +<h3> +<a id="results-achieved" class="anchor" href="#results-achieved" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Results Achieved</h3> +<table> +<thead> +<tr> +<th>Parameter</th> +<th>Results</th> +</tr> +</thead> +<tbody> +<tr> +<td>Query</td> +<td>< 3 Sec</td> +</tr> +<tr> +<td>Data Loading Speed</td> +<td>40 MB/s Per Node</td> +</tr> +<tr> +<td>Concurrent query performance (20 queries)</td> +<td>< 10 Sec</td> +</tr> +</tbody> +</table> +<h2> +<a id="detailed-queries-in-the-smart-city-scenario" class="anchor" href="#detailed-queries-in-the-smart-city-scenario" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Detailed Queries in the Smart City scenario</h2> +<h3> +<a id="scenario-1" class="anchor" href="#scenario-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Scenario</h3> +<p>User wants to analyse the person/vehicle movement and behavior during a certain time period.This output data needs to be joined with a external table for Human details extraction.The query will be run with different time period as filter to identify potential behavior mismatch.</p> +<h3> +<a id="challenges-1" class="anchor" href="#challenges-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Challenges</h3> +<p>Data generated per day is very huge.Data needs to be loaded multiple times per day to accomodate the incoming data size.</p> +<p>Data Loading done once in 6 hours.</p> +<h3> +<a id="solution-1" class="anchor" href="#solution-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Solution</h3> +<p>Setup a Hadoop + Spark + CarbonData cluster managed by YARN.</p> +<p>Since data needs to be queried for a time period, it was recommended to keep the time column at the beginning of schema.</p> +<p>Use table block size as 512MB.</p> +<p>Use local sort mode.</p> +<p>Apart from these, the following CarbonData configuration was suggested to be configured in the cluster.</p> +<p>Use all columns are no-dictionary as the cardinality is high.</p> +<table> +<thead> +<tr> +<th>Configuration for</th> +<th>Parameter</th> +<th>Value</th> +<th>Description</th> +</tr> +</thead> +<tbody> +<tr> +<td>Data Loading</td> +<td>carbon.graph.rowset.size</td> +<td>100000</td> +<td>Based on the size of each row, this determines the memory required during data loading.Higher value leads to increased memory foot print</td> +</tr> +<tr> +<td>Data Loading</td> +<td>enable.unsafe.sort</td> +<td>TRUE</td> +<td>Temporary data generated during sort is huge which causes GC bottlenecks.Using unsafe reduces the pressure on GC</td> +</tr> +<tr> +<td>Data Loading</td> +<td>enable.offheap.sort</td> +<td>TRUE</td> +<td>Temporary data generated during sort is huge which causes GC bottlenecks.Using offheap reduces the pressure on GC.offheap can be accessed through java unsafe.hence enable.unsafe.sort needs to be true</td> +</tr> +<tr> +<td>Data Loading</td> +<td>offheap.sort.chunk.size.in.mb</td> +<td>128</td> +<td>Size of memory to allocate for sorting.Can increase this based on the memory available</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.number.of.cores.while.loading</td> +<td>12</td> +<td>Higher cores can improve data loading speed</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.sort.size</td> +<td>100000</td> +<td>Number of records to sort at a time.More number of records configured will lead to increased memory foot print</td> +</tr> +<tr> +<td>Data Loading</td> +<td>table_blocksize</td> +<td>512</td> +<td>To efficiently schedule multiple tasks during query.This size depends on data scenario.If data is such that the filters would select less number of blocklets to scan, keeping higher number works well.If the number blocklets to scan is more, better to reduce the size as more tasks can be scheduled in parallel.</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.sort.intermediate.files.limit</td> +<td>100</td> +<td>Increased to 100 as number of cores are more.Can perform merging in backgorund.If less number of files to merge, sort threads would be idle</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.use.local.dir</td> +<td>TRUE</td> +<td>yarn application directory will be usually on a single disk.YARN would be configured with multiple disks to be used as temp or to assign randomly to applications.Using the yarn temp directory will allow carbon to use multiple disks and improve IO performance</td> +</tr> +<tr> +<td>Data Loading</td> +<td>carbon.use.multiple.temp.dir</td> +<td>TRUE</td> +<td>multiple disks to write sort files will lead to better IO and reduce the IO bottleneck</td> +</tr> +<tr> +<td>Data Loading</td> +<td>sort.inmemory.size.in.mb</td> +<td>92160</td> +<td>Memory allocated to do inmemory sorting.When more memory is available in the node, configuring this will retain more sort blocks in memory so that the merge sort is faster due to no/very less IO</td> +</tr> +<tr> +<td>Compaction</td> +<td>carbon.major.compaction.size</td> +<td>921600</td> +<td>Sum of several loads to combine into single segment</td> +</tr> +<tr> +<td>Compaction</td> +<td>carbon.number.of.cores.while.compacting</td> +<td>12</td> +<td>Higher number of cores can improve the compaction speed.Data size is huge.Compaction need to use more threads to speed up the process</td> +</tr> +<tr> +<td>Compaction</td> +<td>carbon.enable.auto.load.merge</td> +<td>FALSE</td> +<td>Doing auto minor compaction is costly process as data size is huge.Perform manual compaction when the cluster is less loaded</td> +</tr> +<tr> +<td>Query</td> +<td>carbon.enable.vector.reader</td> +<td>true</td> +<td>To fetch results faster, supporting spark vector processing will speed up the query</td> +</tr> +<tr> +<td>Query</td> +<td>enable.unsafe.in.query.procressing</td> +<td>true</td> +<td>Data that needs to be scanned in huge which in turn generates more short lived Java objects.This cause pressure of GC.using unsafe and offheap will reduce the GC overhead</td> +</tr> +<tr> +<td>Query</td> +<td>use.offheap.in.query.processing</td> +<td>true</td> +<td>Data that needs to be scanned in huge which in turn generates more short lived Java objects.This cause pressure of GC.using unsafe and offheap will reduce the GC overhead.offheap can be accessed through java unsafe.hence enable.unsafe.in.query.procressing needs to be true</td> +</tr> +<tr> +<td>Query</td> +<td>enable.unsafe.columnpage</td> +<td>TRUE</td> +<td>Keep the column pages in offheap memory so that the memory overhead due to java object is less and also reduces GC pressure.</td> +</tr> +<tr> +<td>Query</td> +<td>carbon.unsafe.working.memory.in.mb</td> +<td>10240</td> +<td>Amount of memory to use for offheap operations.Can increase this memory based on the data size</td> +</tr> +</tbody> +</table> +<h3> +<a id="results-achieved-1" class="anchor" href="#results-achieved-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Results Achieved</h3> +<table> +<thead> +<tr> +<th>Parameter</th> +<th>Results</th> +</tr> +</thead> +<tbody> +<tr> +<td>Query (Time Period spanning 1 segment)</td> +<td>< 10 Sec</td> +</tr> +<tr> +<td>Data Loading Speed</td> +<td>45 MB/s Per Node</td> +</tr> +</tbody> +</table> +<h2> +<a id="olapbi-queries-in-the-webinternet-scenario" class="anchor" href="#olapbi-queries-in-the-webinternet-scenario" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>OLAP/BI Queries in the web/Internet scenario</h2> +<h3> +<a id="scenario-2" class="anchor" href="#scenario-2" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Scenario</h3> +<p>An Internet company wants to analyze the average download speed, kind of handsets used in a particular region/area,kind of Apps being used, what kind of videos are trending in a particular region to enable them to identify the appropriate resolution size of videos to speed up transfer, and perform many more analysis to serve th customers better.</p> +<h3> +<a id="challenges-2" class="anchor" href="#challenges-2" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Challenges</h3> +<p>Since data is being queried by a BI tool, all the queries contain group by, which means CarbonData need to return more records as limit cannot be pushed down to carbondata layer.</p> +<p>Results have to be returned faster as the BI tool would not respond till the data is fetched, causing bad user experience.</p> +<p>Data might be loaded less frequently(once or twice in a day), but raw data size is huge, which causes the group by queries to run slower.</p> +<p>Concurrent queries can be more due to the BI dashboard</p> +<h3> +<a id="goal" class="anchor" href="#goal" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Goal</h3> +<ol> +<li>Aggregation queries are faster</li> +<li>Concurrency is high(Number of concurrent queries supported)</li> +</ol> +<h3> +<a id="solution-2" class="anchor" href="#solution-2" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Solution</h3> +<ul> +<li>Use table block size as 128MB so that pruning is more effective</li> +<li>Use global sort mode so that the data to be fetched are grouped together</li> +<li>Create pre-aggregate tables for non timestamp based group by queries</li> +<li>For queries containing group by date, create timeseries based Datamap(pre-aggregate) tables so that the data is rolled up during creation and fetch is faster</li> +<li>Reduce the Spark shuffle partitions.(In our configuration on 14 node cluster, it was reduced to 35 from default of 200)</li> +<li>Enable global dictionary for columns which have less cardinalities.Aggregation can be done on encoded data, there by improving the performance</li> +<li>For columns whose cardinality is high,enable the local dictionary so that store size is less and can take dictionary benefit for scan</li> +</ul> +<h2> +<a id="handling-near-realtime-data-ingestion-scenario" class="anchor" href="#handling-near-realtime-data-ingestion-scenario" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Handling near realtime data ingestion scenario</h2> +<h3> +<a id="scenario-3" class="anchor" href="#scenario-3" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Scenario</h3> +<p>Need to support storing of continously arriving data and make it available immediately for query.</p> +<h3> +<a id="challenges-3" class="anchor" href="#challenges-3" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Challenges</h3> +<p>When the data ingestion is near real time and the data needs to be available for query immediately, usual scenario is to do data loading in micro batches.But this causes the problem of generating many small files.This poses two problems:</p> +<ol> +<li>Small file handling in HDFS is inefficient</li> +<li>CarbonData will suffer in query performance as all the small files will have to be queried when filter is on non time column</li> +</ol> +<p>CarbonData will suffer in query performance as all the small files will have to be queried when filter is on non time column.</p> +<p>Since data is continouly arriving, allocating resources for compaction might not be feasible.</p> +<h3> +<a id="goal-1" class="anchor" href="#goal-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Goal</h3> +<ol> +<li>Data is available in near real time for query as it arrives</li> +<li>CarbonData doesnt suffer from small files problem</li> +</ol> +<h3> +<a id="solution-3" class="anchor" href="#solution-3" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Solution</h3> +<ul> +<li>Use Streaming tables support of CarbonData</li> +<li>Configure the carbon.streaming.segment.max.size property to higher value(default is 1GB) if a bit slower query performance is not a concern</li> +<li>Configure carbon.streaming.auto.handoff.enabled to true so that after the carbon.streaming.segment.max.size is reached, the segment is converted into format optimized for query</li> +<li>Disable auto compaction.Manually trigger the minor compaction with default 4,3 when the cluster is not busy</li> +<li>Manually trigger Major compaction based on the size of segments and the frequency with which the segments are being created</li> +<li>Enable local dictionary</li> +</ul> +<script> +// Show selected style on nav item +$(function() { $('.b-nav__uses').addClass('selected'); }); +</script></div> +</div> +</div> +</div> +<div class="doc-footer"> + <a href="#top" class="scroll-top">Top</a> +</div> +</div> +</section> +</div> +</div> +</div> +</section><!-- End systemblock part --> +<script src="js/custom.js"></script> +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/content/useful-tips-on-carbondata.html ---------------------------------------------------------------------- diff --git a/content/useful-tips-on-carbondata.html b/content/useful-tips-on-carbondata.html deleted file mode 100644 index eba1b0a..0000000 --- a/content/useful-tips-on-carbondata.html +++ /dev/null @@ -1,542 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - <title>CarbonData</title> - <style> - - </style> - <!-- Bootstrap --> - - <link rel="stylesheet" href="css/bootstrap.min.css"> - <link href="css/style.css" rel="stylesheet"> - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js";></script> - <![endif]--> - <script src="js/jquery.min.js"></script> - <script src="js/bootstrap.min.js"></script> - <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js";></script> - - -</head> -<body> -<header> - <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper"> - <div class="container"> - <div class="navbar-header"> - <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse" - class="navbar-toggle collapsed" type="button"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a href="index.html" class="logo"> - <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/> - </a> - </div> - <div class="navbar-collapse collapse cd_navcontnt" id="navbar"> - <ul class="nav navbar-nav navbar-right navlist-custom"> - <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a> - </li> - <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true" - aria-expanded="false"> Download <span class="caret"></span></a> - <ul class="dropdown-menu"> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; - target="_blank">Apache CarbonData 1.4.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; - target="_blank">Apache CarbonData 1.4.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"; - target="_blank">Apache CarbonData 1.3.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; - target="_blank">Apache CarbonData 1.3.0</a></li> - <li> - <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; - target="_blank">Release Archive</a></li> - </ul> - </li> - <li><a href="documentation.html" class="active">Documentation</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" - aria-expanded="false">Community <span class="caret"></span></a> - <ul class="dropdown-menu"> - <li> - <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"; - target="_blank">Contributing to CarbonData</a></li> - <li> - <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"; - target="_blank">Release Guide</a></li> - <li> - <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"; - target="_blank">Project PMC and Committers</a></li> - <li> - <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"; - target="_blank">CarbonData Meetups</a></li> - <li><a href="security.html">Apache CarbonData Security</a></li> - <li><a href="https://issues.apache.org/jira/browse/CARBONDATA"; target="_blank">Apache - Jira</a></li> - <li><a href="videogallery.html">CarbonData Videos </a></li> - </ul> - </li> - <li class="dropdown"> - <a href="http://www.apache.org/"; class="apache_link hidden-xs dropdown-toggle" - data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> - <ul class="dropdown-menu"> - <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> - <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> - <li><a href="http://www.apache.org/foundation/sponsorship.html"; - target="_blank">Sponsorship</a></li> - <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> - </ul> - </li> - - <li class="dropdown"> - <a href="http://www.apache.org/"; class="hidden-lg hidden-md hidden-sm dropdown-toggle" - data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> - <ul class="dropdown-menu"> - <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> - <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> - <li><a href="http://www.apache.org/foundation/sponsorship.html"; - target="_blank">Sponsorship</a></li> - <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> - </ul> - </li> - - <li> - <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a> - - </li> - - </ul> - </div><!--/.nav-collapse --> - <div id="search-box"> - <form method="get" action="http://www.google.com/search"; target="_blank"> - <div class="search-block"> - <table border="0" cellpadding="0" width="100%"> - <tr> - <td style="width:80%"> - <input type="text" name="q" size=" 5" maxlength="255" value="" - class="search-input" placeholder="Search...." required/> - </td> - <td style="width:20%"> - <input type="submit" value="Search"/></td> - </tr> - <tr> - <td align="left" style="font-size:75%" colspan="2"> - <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/> - <span style=" position: relative; top: -3px;"> Only search for CarbonData</span> - </td> - </tr> - </table> - </div> - </form> - </div> - </div> - </nav> -</header> <!-- end Header part --> - -<div class="fixed-padding"></div> <!-- top padding with fixde header --> - -<section><!-- Dashboard nav --> - <div class="container-fluid q"> - <div class="col-sm-12 col-md-12 maindashboard"> - <div class="verticalnavbar"> - <nav class="b-sticky-nav"> - <div class="nav-scroller"> - <div class="nav__inner"> - <a class="b-nav__home nav__item" href="/">home</a> - <a class="b-nav__intro nav__item" href="/intro">introduction</a> - <a class="b-nav__quickstart nav__item" href="/quickstart">quickstart</a> - <a class="b-nav__uses nav__item" href="/uses">use cases</a> - - <div class="nav__item nav__item__with__subs"> - <a class="b-nav__docs nav__item nav__sub__anchor" href="/documentation">documentation</a> - <a class="nav__item nav__sub__item" href="/documentation#gettingStarted">getting started</a> - <a class="nav__item nav__sub__item" href="/documentation#api">APIs</a> - <a class="b-nav__streams nav__item nav__sub__item" href="/documentation/streams">kafka streams</a> - <a class="nav__item nav__sub__item" href="/documentation#connect">kafka connect</a> - <a class="nav__item nav__sub__item" href="/documentation#configuration">configuration</a> - <a class="nav__item nav__sub__item" href="/documentation#design">design</a> - <a class="nav__item nav__sub__item" href="/documentation#implementation">implementation</a> - <a class="nav__item nav__sub__item" href="/documentation#operations">operations</a> - <a class="nav__item nav__sub__item" href="/documentation#security">security</a> - </div> - - <a class="b-nav__performance nav__item" href="/performance">performance</a> - <a class="b-nav__poweredby nav__item" href="/powered-by">powered by</a> - <a class="b-nav__project nav__item" href="/project">project info</a> - <a class="b-nav__ecosystem nav__item" href="https://cwiki.apache.org/confluence/display/KAFKA/Ecosystem"; target="_blank">ecosystem</a> - <a class="b-nav__clients nav__item" href="https://cwiki.apache.org/confluence/display/KAFKA/Clients"; target="_blank">clients</a> - <a class="b-nav__events nav__item" href="/events">events</a> - <a class="b-nav__contact nav__item" href="/contact">contact us</a> - - <div class="nav__item nav__item__with__subs"> - <a class="b-nav__apache nav__item nav__sub__anchor b-nav__sub__anchor" href="#">apache</a> - <a class="b-nav__apache nav__item nav__sub__item" href="http://www.apache.org/"; target="_blank">foundation</a> - <a class="b-nav__apache nav__item nav__sub__item" href="http://www.apache.org/licenses/"; target="_blank">license</a> - <a class="b-nav__apache nav__item nav__sub__item" href="http://www.apache.org/foundation/sponsorship.html"; target="_blank">sponsorship</a> - <a class="b-nav__apache nav__item nav__sub__item" href="http://www.apache.org/foundation/thanks.html"; target="_blank">thanks</a> - <a class="b-nav__apache nav__item nav__sub__item" href="http://www.apache.org/security/"; target="_blank">security</a> - </div> - - <a class="btn" href="/downloads">download</a> - <div class="social-links"> - <a class="twitter" href="https://twitter.com/apachekafka"; target="_blank">@apachekafka</a> - </div> - </div> - </div> - <div class="navindicator"> - <div class="b-nav__home navindicator__item"></div> - <div class="b-nav__intro navindicator__item"></div> - <div class="b-nav__quickstart navindicator__item"></div> - <div class="b-nav__uses navindicator__item"></div> - <div class="b-nav__docs navindicator__item"></div> - <div class="b-nav__performance navindicator__item"></div> - <div class="b-nav__poweredby navindicator__item"></div> - <div class="b-nav__project navindicator__item"></div> - <div class="b-nav__ecosystem navindicator__item"></div> - <div class="b-nav__clients navindicator__item"></div> - <div class="b-nav__events navindicator__item"></div> - <div class="b-nav__contact navindicator__item"></div> - </div> - </nav> - </div> - <div class="mdcontent"> - <section> - <div style="padding:10px 15px;"> - <div id="viewpage" name="viewpage"> - <div class="row"> - <div class="col-sm-12 col-md-12"> - <div> -<h1> -<a id="useful-tips" class="anchor" href="#useful-tips" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Useful Tips</h1> -<p>This tutorial guides you to create CarbonData Tables and optimize performance. -The following sections will elaborate on the below topics :</p> -<ul> -<li><a href="#suggestions-to-create-carbondata-table">Suggestions to create CarbonData Table</a></li> -<li><a href="#configuration-for-optimizing-data-loading-performance-for-massive-data">Configuration for Optimizing Data Loading performance for Massive Data</a></li> -<li><a href="#configurations-for-optimizing-carbondata-performance">Optimizing Mass Data Loading</a></li> -</ul> -<h2> -<a id="suggestions-to-create-carbondata-table" class="anchor" href="#suggestions-to-create-carbondata-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Suggestions to Create CarbonData Table</h2> -<p>For example, the results of the analysis for table creation with dimensions ranging from 10 thousand to 10 billion rows and 100 to 300 columns have been summarized below. -The following table describes some of the columns from the table used.</p> -<ul> -<li><strong>Table Column Description</strong></li> -</ul> -<table> -<thead> -<tr> -<th>Column Name</th> -<th>Data Type</th> -<th>Cardinality</th> -<th>Attribution</th> -</tr> -</thead> -<tbody> -<tr> -<td>msisdn</td> -<td>String</td> -<td>30 million</td> -<td>Dimension</td> -</tr> -<tr> -<td>BEGIN_TIME</td> -<td>BigInt</td> -<td>10 Thousand</td> -<td>Dimension</td> -</tr> -<tr> -<td>HOST</td> -<td>String</td> -<td>1 million</td> -<td>Dimension</td> -</tr> -<tr> -<td>Dime_1</td> -<td>String</td> -<td>1 Thousand</td> -<td>Dimension</td> -</tr> -<tr> -<td>counter_1</td> -<td>Decimal</td> -<td>NA</td> -<td>Measure</td> -</tr> -<tr> -<td>counter_2</td> -<td>Numeric(20,0)</td> -<td>NA</td> -<td>Measure</td> -</tr> -<tr> -<td>...</td> -<td>...</td> -<td>NA</td> -<td>Measure</td> -</tr> -<tr> -<td>counter_100</td> -<td>Decimal</td> -<td>NA</td> -<td>Measure</td> -</tr> -</tbody> -</table> -<ul> -<li><strong>Put the frequently-used column filter in the beginning</strong></li> -</ul> -<p>For example, MSISDN filter is used in most of the query then we must put the MSISDN in the first column. -The create table command can be modified as suggested below :</p> -<pre><code>create table carbondata_table( - msisdn String, - BEGIN_TIME bigint, - HOST String, - Dime_1 String, - counter_1, Decimal - ... - - )STORED BY 'carbondata' - TBLPROPERTIES ('SORT_COLUMNS'='msisdn, Dime_1') -</code></pre> -<p>Now the query with MSISDN in the filter will be more efficient.</p> -<ul> -<li><strong>Put the frequently-used columns in the order of low to high cardinality</strong></li> -</ul> -<p>If the table in the specified query has multiple columns which are frequently used to filter the results, it is suggested to put -the columns in the order of cardinality low to high. This ordering of frequently used columns improves the compression ratio and -enhances the performance of queries with filter on these columns.</p> -<p>For example, if MSISDN, HOST and Dime_1 are frequently-used columns, then the column order of table is suggested as -Dime_1>HOST>MSISDN, because Dime_1 has the lowest cardinality. -The create table command can be modified as suggested below :</p> -<pre><code>create table carbondata_table( - msisdn String, - BEGIN_TIME bigint, - HOST String, - Dime_1 String, - counter_1, Decimal - ... - - )STORED BY 'carbondata' - TBLPROPERTIES ('SORT_COLUMNS'='Dime_1, HOST, MSISDN') -</code></pre> -<ul> -<li><strong>For measure type columns with non high accuracy, replace Numeric(20,0) data type with Double data type</strong></li> -</ul> -<p>For columns of measure type, not requiring high accuracy, it is suggested to replace Numeric data type with Double to enhance query performance. -The create table command can be modified as below :</p> -<pre><code> create table carbondata_table( - Dime_1 String, - BEGIN_TIME bigint, - END_TIME bigint, - HOST String, - MSISDN String, - counter_1 decimal, - counter_2 double, - ... - )STORED BY 'carbondata' - TBLPROPERTIES ('SORT_COLUMNS'='Dime_1, HOST, MSISDN') -</code></pre> -<p>The result of performance analysis of test-case shows reduction in query execution time from 15 to 3 seconds, thereby improving performance by nearly 5 times.</p> -<ul> -<li><strong>Columns of incremental character should be re-arranged at the end of dimensions</strong></li> -</ul> -<p>Consider the following scenario where data is loaded each day and the begin_time is incremental for each load, it is suggested to put begin_time at the end of dimensions. -Incremental values are efficient in using min/max index. The create table command can be modified as below :</p> -<pre><code>create table carbondata_table( - Dime_1 String, - HOST String, - MSISDN String, - counter_1 double, - counter_2 double, - BEGIN_TIME bigint, - END_TIME bigint, - ... - counter_100 double - )STORED BY 'carbondata' - TBLPROPERTIES ('SORT_COLUMNS'='Dime_1, HOST, MSISDN') -</code></pre> -<p><strong>NOTE:</strong></p> -<ul> -<li>BloomFilter can be created to enhance performance for queries with precise equal/in conditions. You can find more information about it in BloomFilter datamap <a href="https://github.com/apache/carbondata/blob/master/docs/datamap/bloomfilter-datamap-guide.html"; target=_blank>document</a>.</li> -</ul> -<h2> -<a id="configuration-for-optimizing-data-loading-performance-for-massive-data" class="anchor" href="#configuration-for-optimizing-data-loading-performance-for-massive-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Configuration for Optimizing Data Loading performance for Massive Data</h2> -<p>CarbonData supports large data load, in this process sorting data while loading consumes a lot of memory and disk IO and -this can result sometimes in "Out Of Memory" exception. -If you do not have much memory to use, then you may prefer to slow the speed of data loading instead of data load failure. -You can configure CarbonData by tuning following properties in carbon.properties file to get a better performance.</p> -<table> -<thead> -<tr> -<th>Parameter</th> -<th>Default Value</th> -<th>Description/Tuning</th> -</tr> -</thead> -<tbody> -<tr> -<td>carbon.number.of.cores.while.loading</td> -<td>Default: 2.This value should be >= 2</td> -<td>Specifies the number of cores used for data processing during data loading in CarbonData.</td> -</tr> -<tr> -<td>carbon.sort.size</td> -<td>Default: 100000. The value should be >= 100.</td> -<td>Threshold to write local file in sort step when loading data</td> -</tr> -<tr> -<td>carbon.sort.file.write.buffer.size</td> -<td>Default: 50000.</td> -<td>DataOutputStream buffer.</td> -</tr> -<tr> -<td>carbon.number.of.cores.block.sort</td> -<td>Default: 7</td> -<td>If you have huge memory and CPUs, increase it as you will</td> -</tr> -<tr> -<td>carbon.merge.sort.reader.thread</td> -<td>Default: 3</td> -<td>Specifies the number of cores used for temp file merging during data loading in CarbonData.</td> -</tr> -<tr> -<td>carbon.merge.sort.prefetch</td> -<td>Default: true</td> -<td>You may want set this value to false if you have not enough memory</td> -</tr> -</tbody> -</table> -<p>For example, if there are 10 million records, and i have only 16 cores, 64GB memory, will be loaded to CarbonData table. -Using the default configuration always fail in sort step. Modify carbon.properties as suggested below:</p> -<pre><code>carbon.number.of.cores.block.sort=1 -carbon.merge.sort.reader.thread=1 -carbon.sort.size=5000 -carbon.sort.file.write.buffer.size=5000 -carbon.merge.sort.prefetch=false -</code></pre> -<h2> -<a id="configurations-for-optimizing-carbondata-performance" class="anchor" href="#configurations-for-optimizing-carbondata-performance" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Configurations for Optimizing CarbonData Performance</h2> -<p>Recently we did some performance POC on CarbonData for Finance and telecommunication Field. It involved detailed queries and aggregation -scenarios. After the completion of POC, some of the configurations impacting the performance have been identified and tabulated below :</p> -<table> -<thead> -<tr> -<th>Parameter</th> -<th>Location</th> -<th>Used For</th> -<th>Description</th> -<th>Tuning</th> -</tr> -</thead> -<tbody> -<tr> -<td>carbon.sort.intermediate.files.limit</td> -<td>spark/carbonlib/carbon.properties</td> -<td>Data loading</td> -<td>During the loading of data, local temp is used to sort the data. This number specifies the minimum number of intermediate files after which the merge sort has to be initiated.</td> -<td>Increasing the parameter to a higher value will improve the load performance. For example, when we increase the value from 20 to 100, it increases the data load performance from 35MB/S to more than 50MB/S. Higher values of this parameter consumes more memory during the load.</td> -</tr> -<tr> -<td>carbon.number.of.cores.while.loading</td> -<td>spark/carbonlib/carbon.properties</td> -<td>Data loading</td> -<td>Specifies the number of cores used for data processing during data loading in CarbonData.</td> -<td>If you have more number of CPUs, then you can increase the number of CPUs, which will increase the performance. For example if we increase the value from 2 to 4 then the CSV reading performance can increase about 1 times</td> -</tr> -<tr> -<td>carbon.compaction.level.threshold</td> -<td>spark/carbonlib/carbon.properties</td> -<td>Data loading and Querying</td> -<td>For minor compaction, specifies the number of segments to be merged in stage 1 and number of compacted segments to be merged in stage 2.</td> -<td>Each CarbonData load will create one segment, if every load is small in size it will generate many small file over a period of time impacting the query performance. Configuring this parameter will merge the small segment to one big segment which will sort the data and improve the performance. For Example in one telecommunication scenario, the performance improves about 2 times after minor compaction.</td> -</tr> -<tr> -<td>spark.sql.shuffle.partitions</td> -<td>spark/conf/spark-defaults.conf</td> -<td>Querying</td> -<td>The number of task started when spark shuffle.</td> -<td>The value can be 1 to 2 times as much as the executor cores. In an aggregation scenario, reducing the number from 200 to 32 reduced the query time from 17 to 9 seconds.</td> -</tr> -<tr> -<td>spark.executor.instances/spark.executor.cores/spark.executor.memory</td> -<td>spark/conf/spark-defaults.conf</td> -<td>Querying</td> -<td>The number of executors, CPU cores, and memory used for CarbonData query.</td> -<td>In the bank scenario, we provide the 4 CPUs cores and 15 GB for each executor which can get good performance. This 2 value does not mean more the better. It needs to be configured properly in case of limited resources. For example, In the bank scenario, it has enough CPU 32 cores each node but less memory 64 GB each node. So we cannot give more CPU but less memory. For example, when 4 cores and 12GB for each executor. It sometimes happens GC during the query which impact the query performance very much from the 3 second to more than 15 seconds. In this scenario need to increase the memory or decrease the CPU cores.</td> -</tr> -<tr> -<td>carbon.detail.batch.size</td> -<td>spark/carbonlib/carbon.properties</td> -<td>Data loading</td> -<td>The buffer size to store records, returned from the block scan.</td> -<td>In limit scenario this parameter is very important. For example your query limit is 1000. But if we set this value to 3000 that means we get 3000 records from scan but spark will only take 1000 rows. So the 2000 remaining are useless. In one Finance test case after we set it to 100, in the limit 1000 scenario the performance increase about 2 times in comparison to if we set this value to 12000.</td> -</tr> -<tr> -<td>carbon.use.local.dir</td> -<td>spark/carbonlib/carbon.properties</td> -<td>Data loading</td> -<td>Whether use YARN local directories for multi-table load disk load balance</td> -<td>If this is set it to true CarbonData will use YARN local directories for multi-table load disk load balance, that will improve the data load performance.</td> -</tr> -<tr> -<td>carbon.use.multiple.temp.dir</td> -<td>spark/carbonlib/carbon.properties</td> -<td>Data loading</td> -<td>Whether to use multiple YARN local directories during table data loading for disk load balance</td> -<td>After enabling 'carbon.use.local.dir', if this is set to true, CarbonData will use all YARN local directories during data load for disk load balance, that will improve the data load performance. Please enable this property when you encounter disk hotspot problem during data loading.</td> -</tr> -<tr> -<td>carbon.sort.temp.compressor</td> -<td>spark/carbonlib/carbon.properties</td> -<td>Data loading</td> -<td>Specify the name of compressor to compress the intermediate sort temporary files during sort procedure in data loading.</td> -<td>The optional values are 'SNAPPY','GZIP','BZIP2','LZ4','ZSTD' and empty. By default, empty means that Carbondata will not compress the sort temp files. This parameter will be useful if you encounter disk bottleneck.</td> -</tr> -<tr> -<td>carbon.load.skewedDataOptimization.enabled</td> -<td>spark/carbonlib/carbon.properties</td> -<td>Data loading</td> -<td>Whether to enable size based block allocation strategy for data loading.</td> -<td>When loading, carbondata will use file size based block allocation strategy for task distribution. It will make sure that all the executors process the same size of data -- It's useful if the size of your input data files varies widely, say 1MB~1GB.</td> -</tr> -<tr> -<td>carbon.load.min.size.enabled</td> -<td>spark/carbonlib/carbon.properties</td> -<td>Data loading</td> -<td>Whether to enable node minumun input data size allocation strategy for data loading.</td> -<td>When loading, carbondata will use node minumun input data size allocation strategy for task distribution. It will make sure the node load the minimum amount of data -- It's useful if the size of your input data files very small, say 1MB~256MB,Avoid generating a large number of small files.</td> -</tr> -</tbody> -</table> -<p>Note: If your CarbonData instance is provided only for query, you may specify the property 'spark.speculation=true' which is in conf directory of spark.</p> -</div> -</div> -</div> -</div> -<div class="doc-footer"> - <a href="#top" class="scroll-top">Top</a> -</div> -</div> -</section> -</div> -</div> -</div> -</section><!-- End systemblock part --> -<script src="js/custom.js"></script> -</body> -</html> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/content/videogallery.html ---------------------------------------------------------------------- diff --git a/content/videogallery.html b/content/videogallery.html index e494174..c282364 100644 --- a/content/videogallery.html +++ b/content/videogallery.html @@ -246,4 +246,4 @@ </body> -</html> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index af4f234..ba425e5 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -15,7 +15,8 @@ fileList=["configuration-parameters", "segment-management-on-carbondata", "release-guide", "how-to-contribute-to-apache-carbondata", - "introduction" + "introduction", + "usecases" ] dataMapFileList=[ "bloomfilter-datamap-guide", @@ -34,4 +35,20 @@ outputFileLocation="src/main/webapp/" apiUrl="https://raw.githubusercontent.com/apache/carbondata/master/docs/"; mdLink="https://api.github.com/markdown/raw"; mdFileLocation="src/site/markdown/" -imagesFilesList=["file-structure-of-carbondata"] \ No newline at end of file +imagesFilesList=["file-structure-of-carbondata", +"2-1_1", +"2-2_1", +"2-3_1", +"2-3_2", +"2-3_3", +"2-3_4", +"2-4_1", +"2-5_1", +"2-5_2", +"2-5_3", +"2-6_1", +"carbon_data_file_structure_new", +"carbon_data_format_new", +"carbondata-performance" +] +scriptsPath="src/main/scala/scripts/" \ No newline at end of file
