http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/index.html ---------------------------------------------------------------------- diff --git a/content/index.html b/content/index.html index 3fe3ba4..e4e0c0e 100644 --- a/content/index.html +++ b/content/index.html @@ -54,6 +54,9 @@ class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -63,27 +66,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> @@ -328,6 +310,13 @@ </h4> <div class="linkblock"> <div class="block-row"> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a> + <span class="release-date">Aug 2018</span> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.4.1+Release"; + class="whatsnew" target="_blank">what's new</a> + </div> + <div class="block-row"> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a> <span class="release-date">May 2018</span> @@ -348,21 +337,6 @@ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.3.0+Release"; class="whatsnew" target="_blank">what's new</a> </div> - <div class="block-row"> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a> - <span class="release-date">Sept 2017</span> - <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.2.0+Release"; - class="whatsnew" target="_blank">what's new</a> - </div> - <div class="block-row"> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a> - <span class="release-date">July 2017</span> - <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.1.1+Release"; - class="whatsnew" target="_blank">what's new</a> - </div> - </div> </div> <div class="nextR"> <h4 class="title">Release Notes @@ -389,10 +363,6 @@ class="fa fa-github icon-margin-r" aria-hidden="true"></i> CarbonData Github </a> - <a href="pdf/CarbonData Documentation.pdf" class="quickstart-btn" - target="_blank"><i class="fa fa-file-pdf-o icon-margin-r" - aria-hidden="true"></i></i> Download Documentation - </a> <a href="pdf/CarbonData-TPCH-Report.pdf" class="quickstart-btn" target="_blank"><i class="fa fa-file-pdf-o icon-margin-r" aria-hidden="true"></i></i> CarbonData TPCH Report @@ -498,7 +468,7 @@ to do is:</p> <ol class="orderlist"> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"; + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; target="_blank">Download</a>the latest release. </li> @@ -689,7 +659,7 @@ <div class="bottomcontant"> <div class="container-fluid"> <div class="col-md-8 col-sm-8"> - <p class="copyright-txt">Copyright © 2016-2017 Apache CarbonData. All rights reserved | + <p class="copyright-txt">Copyright © 2017-2018 Apache CarbonData. All rights reserved | <a href="http://www.apache.org/"; class="term-links" target="_blank">Apache Software Foundation </a> | <a href="http://www.apache.org/foundation/policies/privacy.html";
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/installation-guide.html ---------------------------------------------------------------------- diff --git a/content/installation-guide.html b/content/installation-guide.html index ffb750e..2e7fab6 100644 --- a/content/installation-guide.html +++ b/content/installation-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/lucene-datamap-guide.html ---------------------------------------------------------------------- diff --git a/content/lucene-datamap-guide.html b/content/lucene-datamap-guide.html index 5f56429..7b1598e 100644 --- a/content/lucene-datamap-guide.html +++ b/content/lucene-datamap-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> @@ -176,8 +158,9 @@ <div id="viewpage" name="viewpage"> <div class="row"> <div class="col-sm-12 col-md-12"> - <div><h1> -<a id="carbondata-lucene-datamap-alpha-feature-in-140" class="anchor" href="#carbondata-lucene-datamap-alpha-feature-in-140" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Lucene DataMap (Alpha feature in 1.4.0)</h1> + <div> +<h1> +<a id="carbondata-lucene-datamap-alpha-feature" class="anchor" href="#carbondata-lucene-datamap-alpha-feature" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Lucene DataMap (Alpha Feature)</h1> <ul> <li><a href="#datamap-management">DataMap Management</a></li> <li><a href="#lucene-datamap-introduction">Lucene Datamap</a></li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/mainpage.html ---------------------------------------------------------------------- diff --git a/content/mainpage.html b/content/mainpage.html index 259d485..d515853 100644 --- a/content/mainpage.html +++ b/content/mainpage.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> @@ -198,9 +180,11 @@ <li><a href="configuration-parameters.html">Configuring CarbonData</a></li> <li><a href="streaming-guide.html">Streaming Guide</a></li> <li><a href="sdk-guide.html">SDK Guide</a></li> + <li><a href="s3-guide.html">S3 Guide (Alpha Feature)</a></li> <li><a href="datamap-developer-guide.html">DataMap Developer Guide</a></li> - <li><a href="bloomfilter-datamap-guide.html">CarbonData BloomFilter DataMap (Alpha feature in 1.4.0)</a></li> - <li><a href="lucene-datamap-guide.html">CarbonData Lucene DataMap (Alpha feature in 1.4.0)</a></li> + <li><a href="datamap-management.html">CarbonData DataMap Management</a></li> + <li><a href="bloomfilter-datamap-guide.html">CarbonData BloomFilter DataMap (Alpha Feature)</a></li> + <li><a href="lucene-datamap-guide.html">CarbonData Lucene DataMap (Alpha Feature)</a></li> <li><a href="preaggregate-datamap-guide.html">CarbonData Pre-aggregate DataMap</a></li> <li><a href="timeseries-datamap-guide.html">CarbonData Timeseries DataMap</a></li> <li><a href="faq.html">FAQs</a></li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/pdf/CarbonData Documentation.pdf ---------------------------------------------------------------------- diff --git a/content/pdf/CarbonData Documentation.pdf b/content/pdf/CarbonData Documentation.pdf index c4d389d..45ce585 100644 Binary files a/content/pdf/CarbonData Documentation.pdf and b/content/pdf/CarbonData Documentation.pdf differ http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/preaggregate-datamap-guide.html ---------------------------------------------------------------------- diff --git a/content/preaggregate-datamap-guide.html b/content/preaggregate-datamap-guide.html index d195bfe..d68764d 100644 --- a/content/preaggregate-datamap-guide.html +++ b/content/preaggregate-datamap-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> @@ -176,7 +158,8 @@ <div id="viewpage" name="viewpage"> <div class="row"> <div class="col-sm-12 col-md-12"> - <div><h1> + <div> +<h1> <a id="carbondata-pre-aggregate-datamap" class="anchor" href="#carbondata-pre-aggregate-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Pre-aggregate DataMap</h1> <ul> <li><a href="#quick-example">Quick Example</a></li> @@ -422,6 +405,7 @@ change datatype command, CarbonData will check whether it will impact the pre-ag not, the operation is allowed, otherwise operation will be rejected by throwing exception.</li> <li>Partition management command: <code>ALTER TABLE ADD/DROP PARTITION</code> </li> +<li>Complex Datatypes for preaggregate is not supported.</li> </ol> <p>However, there is still way to support these operations on main table, in current CarbonData release, user can do as following:</p> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/quick-start-guide.html ---------------------------------------------------------------------- diff --git a/content/quick-start-guide.html b/content/quick-start-guide.html index 20cfbf1..89380b4 100644 --- a/content/quick-start-guide.html +++ b/content/quick-start-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/s3-guide.html ---------------------------------------------------------------------- diff --git a/content/s3-guide.html b/content/s3-guide.html new file mode 100644 index 0000000..d9edeaf --- /dev/null +++ b/content/s3-guide.html @@ -0,0 +1,239 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>CarbonData</title> + <style> + + </style> + <!-- Bootstrap --> + + <link rel="stylesheet" href="css/bootstrap.min.css"> + <link href="css/style.css" rel="stylesheet"> + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js";></script> + <![endif]--> + <script src="js/jquery.min.js"></script> + <script src="js/bootstrap.min.js"></script> + + +</head> +<body> +<header> + <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper"> + <div class="container"> + <div class="navbar-header"> + <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse" + class="navbar-toggle collapsed" type="button"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a href="index.html" class="logo"> + <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/> + </a> + </div> + <div class="navbar-collapse collapse cd_navcontnt" id="navbar"> + <ul class="nav navbar-nav navbar-right navlist-custom"> + <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a> + </li> + <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false"> Download <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; + target="_blank">Apache CarbonData 1.4.0</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"; + target="_blank">Apache CarbonData 1.3.1</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; + target="_blank">Release Archive</a></li> + </ul> + </li> + <li><a href="mainpage.html" class="active">Documentation</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false">Community <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"; + target="_blank">Contributing to CarbonData</a></li> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"; + target="_blank">Release Guide</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"; + target="_blank">Project PMC and Committers</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"; + target="_blank">CarbonData Meetups</a></li> + <li><a href="security.html">Apache CarbonData Security</a></li> + <li><a href="https://issues.apache.org/jira/browse/CARBONDATA"; target="_blank">Apache + Jira</a></li> + <li><a href="videogallery.html">CarbonData Videos </a></li> + </ul> + </li> + <li class="dropdown"> + <a href="http://www.apache.org/"; class="apache_link hidden-xs dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li class="dropdown"> + <a href="http://www.apache.org/"; class="hidden-lg hidden-md hidden-sm dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li> + <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a> + + </li> + + </ul> + </div><!--/.nav-collapse --> + <div id="search-box"> + <form method="get" action="http://www.google.com/search"; target="_blank"> + <div class="search-block"> + <table border="0" cellpadding="0" width="100%"> + <tr> + <td style="width:80%"> + <input type="text" name="q" size=" 5" maxlength="255" value="" + class="search-input" placeholder="Search...." required/> + </td> + <td style="width:20%"> + <input type="submit" value="Search"/></td> + </tr> + <tr> + <td align="left" style="font-size:75%" colspan="2"> + <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/> + <span style=" position: relative; top: -3px;"> Only search for CarbonData</span> + </td> + </tr> + </table> + </div> + </form> + </div> + </div> + </nav> +</header> <!-- end Header part --> + +<div class="fixed-padding"></div> <!-- top padding with fixde header --> + +<section><!-- Dashboard nav --> + <div class="container-fluid q"> + <div class="col-sm-12 col-md-12 maindashboard"> + <div class="row"> + <section> + <div style="padding:10px 15px;"> + <div id="viewpage" name="viewpage"> + <div class="row"> + <div class="col-sm-12 col-md-12"> + <div> +<p>#S3 Guide (Alpha Feature 1.4.1)</p> +<p>Object storage is the recommended storage format in cloud as it can support storing large data +files. S3 APIs are widely used for accessing object stores. This can be +used to store or retrieve data on Amazon cloud, Huawei Cloud(OBS) or on any other object +stores conforming to S3 API. +Storing data in cloud is advantageous as there are no restrictions on the size of +data and the data can be accessed from anywhere at any time. +Carbondata can support any Object Storage that conforms to Amazon S3 API. +Carbondata relies on Hadoop provided S3 filesystem APIs to access Object stores.</p> +<p>#Writing to Object Storage</p> +<p>To store carbondata files onto Object Store, <code>carbon.storelocation</code> property will have +to be configured with Object Store path in CarbonProperties file.</p> +<p>For example:</p> +<pre><code>carbon.storelocation=s3a://mybucket/carbonstore. +</code></pre> +<p>If the existing store location cannot be changed or only specific tables need to be stored +onto cloud object store, it can be done so by specifying the <code>location</code> option in the create +table DDL command.</p> +<p>For example:</p> +<pre><code>CREATE TABLE IF NOT EXISTS db1.table1(col1 string, col2 int) STORED AS carbondata LOCATION 's3a://mybucket/carbonstore' +</code></pre> +<p>For more details on create table, Refer <a href="./data-management-on-carbondata.html#create-table">data-management-on-carbondata</a></p> +<p>#Authentication</p> +<p>Authentication properties will have to be configured to store the carbondata files on to S3 location.</p> +<p>Authentication properties can be set in any of the following ways:</p> +<ol> +<li> +<p>Set authentication properties in core-site.xml, refer +<a href="https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html#Authentication_properties"; rel="nofollow">hadoop authentication document</a></p> +</li> +<li> +<p>Set authentication properties in spark-defaults.conf.</p> +</li> +</ol> +<p>Example</p> +<pre><code>spark.hadoop.fs.s3a.secret.key=123 +spark.hadoop.fs.s3a.access.key=456 +</code></pre> +<ol start="3"> +<li>Pass authentication properties with spark-submit as configuration.</li> +</ol> +<p>Example:</p> +<pre><code>./bin/spark-submit --master yarn --conf spark.hadoop.fs.s3a.secret.key=123 --conf spark.hadoop.fs +.s3a.access.key=456 --class= +</code></pre> +<ol start="4"> +<li>Set authentication properties to hadoop configuration object in sparkContext.</li> +</ol> +<p>Example:</p> +<pre><code>sparkSession.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", "123") +sparkSession.sparkContext.hadoopConfiguration.set("fs.s3a.access.key","456") +</code></pre> +<p>#Recommendations</p> +<ol> +<li>Object Storage like S3 does not support file leasing mechanism(supported by HDFS) that is +required to take locks which ensure consistency between concurrent operations therefore, it is +recommended to set the configurable lock path property(<a href="https://github.com/apache/carbondata/blob/master/docs/configuration-parameters.html#miscellaneous-configuration";>carbon.lock.path</a>) +to a HDFS directory.</li> +<li>Concurrent data manipulation operations are not supported. Object stores follow eventual +consistency semantics, i.e., any put request might take some time to reflect when trying to list +.This behaviour causes not to ensure the data read is always consistent or latest.</li> +</ol> +</div> +</div> +</div> +</div> +<div class="doc-footer"> + <a href="#top" class="scroll-top">Top</a> +</div> +</div> +</section> +</div> +</div> +</div> +</section><!-- End systemblock part --> +<script src="js/custom.js"></script> +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/sdk-guide.html ---------------------------------------------------------------------- diff --git a/content/sdk-guide.html b/content/sdk-guide.html index d980f83..cc55dc0 100644 --- a/content/sdk-guide.html +++ b/content/sdk-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> @@ -176,7 +158,8 @@ <div id="viewpage" name="viewpage"> <div class="row"> <div class="col-sm-12 col-md-12"> - <div><h1> + <div> +<h1> <a id="sdk-guide" class="anchor" href="#sdk-guide" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>SDK Guide</h1> <p>In the carbon jars package, there exist a carbondata-store-sdk-x.x.x-SNAPSHOT.jar, including SDK writer and reader.</p> <h1> @@ -286,6 +269,47 @@ These SDK writer output contains just a carbondata and carbonindex files. No met } } }</pre></div> +<h3> +<a id="example-with-json-format" class="anchor" href="#example-with-json-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Example with Json format</h3> +<div class="highlight highlight-source-java"><pre><span class="pl-k">import</span> <span class="pl-smi">java.io.IOException</span>; + +<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException</span>; +<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.core.metadata.datatype.DataTypes</span>; +<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.core.util.CarbonProperties</span>; +<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.CarbonWriter</span>; +<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.CarbonWriterBuilder</span>; +<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.Field</span>; +<span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.sdk.file.Schema</span>; + +<span class="pl-k">public</span> <span class="pl-k">class</span> <span class="pl-en">TestSdkJson</span> { + + <span class="pl-k">public</span> <span class="pl-k">static</span> <span class="pl-k">void</span> <span class="pl-en">main</span>(<span class="pl-k">String</span>[] <span class="pl-v">args</span>) <span class="pl-k">throws</span> <span class="pl-smi">InvalidLoadOptionException</span> { + testJsonSdkWriter(); + } + + <span class="pl-k">public</span> <span class="pl-k">static</span> <span class="pl-k">void</span> <span class="pl-en">testJsonSdkWriter</span>() <span class="pl-k">throws</span> <span class="pl-smi">InvalidLoadOptionException</span> { + <span class="pl-smi">String</span> path <span class="pl-k">=</span> <span class="pl-s"><span class="pl-pds">"</span>./target/testJsonSdkWriter<span class="pl-pds">"</span></span>; + + <span class="pl-k">Field</span>[] fields <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Field</span>[<span class="pl-c1">2</span>]; + fields[<span class="pl-c1">0</span>] <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Field</span>(<span class="pl-s"><span class="pl-pds">"</span>name<span class="pl-pds">"</span></span>, <span class="pl-smi">DataTypes</span><span class="pl-c1"><span class="pl-k">.</span>STRING</span>); + fields[<span class="pl-c1">1</span>] <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Field</span>(<span class="pl-s"><span class="pl-pds">"</span>age<span class="pl-pds">"</span></span>, <span class="pl-smi">DataTypes</span><span class="pl-c1"><span class="pl-k">.</span>INT</span>); + + <span class="pl-smi">Schema</span> <span class="pl-smi">CarbonSchema</span> <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-smi">Schema</span>(fields); + + <span class="pl-smi">CarbonWriterBuilder</span> builder <span class="pl-k">=</span> <span class="pl-smi">CarbonWriter</span><span class="pl-k">.</span>builder()<span class="pl-k">.</span>outputPath(path); + + <span class="pl-c"><span class="pl-c">//</span> initialize json writer with carbon schema</span> + <span class="pl-smi">CarbonWriter</span> writer <span class="pl-k">=</span> builder<span class="pl-k">.</span>buildWriterForJsonInput(<span class="pl-smi">CarbonSchema</span>); + <span class="pl-c"><span class="pl-c">//</span> one row of json Data as String</span> + <span class="pl-smi">String</span> <span class="pl-smi">JsonRow</span> <span class="pl-k">=</span> <span class="pl-s"><span class="pl-pds">"</span>{<span class="pl-cce">\"</span>name<span class="pl-cce">\"</span>:<span class="pl-cce">\"</span>abcd<span class="pl-cce">\"</span>, <span class="pl-cce">\"</span>age<span class="pl-cce">\"</span>:10}<span class="pl-pds">"</span></span>; + + <span class="pl-k">int</span> rows <span class="pl-k">=</span> <span class="pl-c1">5</span>; + <span class="pl-k">for</span> (<span class="pl-k">int</span> i <span class="pl-k">=</span> <span class="pl-c1">0</span>; i <span class="pl-k"><</span> rows; i<span class="pl-k">++</span>) { + writer<span class="pl-k">.</span>write(<span class="pl-smi">JsonRow</span>); + } + writer<span class="pl-k">.</span>close(); + } +} </pre></div> <h2> <a id="datatypes-mapping" class="anchor" href="#datatypes-mapping" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Datatypes Mapping</h2> <p>Each of SQL data types are mapped into data types of SDK. Following are the mapping:</p> @@ -339,6 +363,20 @@ These SDK writer output contains just a carbondata and carbonindex files. No met </tr> </tbody> </table> +<p><strong>NOTE:</strong> +Carbon Supports below logical types of AVRO. +a. Date +The date logical type represents a date within the calendar, with no reference to a particular time zone or time of day. +A date logical type annotates an Avro int, where the int stores the number of days from the unix epoch, 1 January 1970 (ISO calendar). +b. Timestamp (millisecond precision) +The timestamp-millis logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one millisecond. +A timestamp-millis logical type annotates an Avro long, where the long stores the number of milliseconds from the unix epoch, 1 January 1970 00:00:00.000 UTC. +c. Timestamp (microsecond precision) +The timestamp-micros logical type represents an instant on the global timeline, independent of a particular time zone or calendar, with a precision of one microsecond. +A timestamp-micros logical type annotates an Avro long, where the long stores the number of microseconds from the unix epoch, 1 January 1970 00:00:00.000000 UTC.</p> +<pre><code>Currently the values of logical types are not validated by carbon. +Expect that avro record passed by the user is already validated by avro record generator tools. +</code></pre> <h2> <a id="run-sql-on-files-directly" class="anchor" href="#run-sql-on-files-directly" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Run SQL on files directly</h2> <p>Instead of creating table and query it, you can also query that file directly with SQL.</p> @@ -396,6 +434,18 @@ public CarbonWriterBuilder withBlockSize(int blockSize); public CarbonWriterBuilder withBlockletSize(int blockletSize); </code></pre> <pre><code>/** + * @param enableLocalDictionary enable local dictionary , default is false + * @return updated CarbonWriterBuilder + */ +public CarbonWriterBuilder enableLocalDictionary(boolean enableLocalDictionary); +</code></pre> +<pre><code>/** + * @param localDictionaryThreshold is localDictionaryThreshold,default is 10000 + * @return updated CarbonWriterBuilder + */ +public CarbonWriterBuilder localDictionaryThreshold(int localDictionaryThreshold) ; +</code></pre> +<pre><code>/** * sets the list of columns that needs to be in sorted order * @param sortColumns is a string array of columns that needs to be sorted. * If it is null or by default all dimensions are selected for sorting @@ -419,7 +469,7 @@ public CarbonWriterBuilder persistSchemaFile(boolean persist); * by default it is system time in nano seconds. * @return updated CarbonWriterBuilder */ -public CarbonWriterBuilder taskNo(String taskNo); +public CarbonWriterBuilder taskNo(long taskNo); </code></pre> <pre><code>/** * To support the load options for sdk writer @@ -458,7 +508,7 @@ public CarbonWriterBuilder withLoadOptions(Map<String, String> options); * @throws IOException * @throws InvalidLoadOptionException */ -public CarbonWriter buildWriterForCSVInput() throws IOException, InvalidLoadOptionException; +public CarbonWriter buildWriterForCSVInput(org.apache.carbondata.sdk.file.Schema schema) throws IOException, InvalidLoadOptionException; </code></pre> <pre><code>/** * Build a {@link CarbonWriter}, which accepts Avro format object @@ -467,14 +517,25 @@ public CarbonWriter buildWriterForCSVInput() throws IOException, InvalidLoadOpti * @throws IOException * @throws InvalidLoadOptionException */ -public CarbonWriter buildWriterForAvroInput() throws IOException, InvalidLoadOptionException; +public CarbonWriter buildWriterForAvroInput(org.apache.avro.Schema schema) throws IOException, InvalidLoadOptionException; +</code></pre> +<pre><code>/** +* Build a {@link CarbonWriter}, which accepts Json object +* @param carbonSchema carbon Schema object +* @return JsonCarbonWriter +* @throws IOException +* @throws InvalidLoadOptionException +*/ +public JsonCarbonWriter buildWriterForJsonInput(Schema carbonSchema); </code></pre> <h3> <a id="class-orgapachecarbondatasdkfilecarbonwriter" class="anchor" href="#class-orgapachecarbondatasdkfilecarbonwriter" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.CarbonWriter</h3> <pre><code>/** * Write an object to the file, the format of the object depends on the implementation -* If AvroCarbonWriter, object is of type org.apache.avro.generic.GenericData.Record -* If CSVCarbonWriter, object is of type String[] +* If AvroCarbonWriter, object is of type org.apache.avro.generic.GenericData.Record, +* which is one row of data. +* If CSVCarbonWriter, object is of type String[], which is one row of data +* If JsonCarbonWriter, object is of type String, which is one row of json * Note: This API is not thread safe * @param object * @throws IOException @@ -624,14 +685,6 @@ External client can make use of this reader to read CarbonData files without Car public CarbonReaderBuilder projection(String[] projectionColumnNames); </code></pre> <pre><code> /** - * Project all Columns for carbon reader - * - * @return CarbonReaderBuilder object - * @throws IOException - */ - public CarbonReaderBuilder projectAllColumns(); -</code></pre> -<pre><code> /** * Configure the transactional status of table * If set to false, then reads the carbondata and carbonindex files from a flat folder structure. * If set to true, then reads the carbondata and carbonindex files from segment folder structure. http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/streaming-guide.html ---------------------------------------------------------------------- diff --git a/content/streaming-guide.html b/content/streaming-guide.html index 981a095..6e2e7ee 100644 --- a/content/streaming-guide.html +++ b/content/streaming-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> @@ -176,7 +158,8 @@ <div id="viewpage" name="viewpage"> <div class="row"> <div class="col-sm-12 col-md-12"> - <div><h1> + <div> +<h1> <a id="carbondata-streaming-ingestion" class="anchor" href="#carbondata-streaming-ingestion" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Streaming Ingestion</h1> <h2> <a id="quick-example" class="anchor" href="#quick-example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Quick example</h2> @@ -198,6 +181,7 @@ <span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">spark</span>.<span class="pl-en">sql</span>.<span class="pl-en">CarbonSession</span>.<span class="pl-en">_</span> <span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">spark</span>.<span class="pl-en">sql</span>.<span class="pl-en">streaming</span>.{<span class="pl-en">ProcessingTime</span>, <span class="pl-en">StreamingQuery</span>} <span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">carbondata</span>.<span class="pl-en">core</span>.<span class="pl-en">util</span>.<span class="pl-en">path</span>.<span class="pl-en">CarbonTablePath</span> + <span class="pl-k">import</span> <span class="pl-en">org</span>.<span class="pl-en">apache</span>.<span class="pl-en">carbondata</span>.<span class="pl-en">streaming</span>.<span class="pl-en">parser</span>.<span class="pl-en">CarbonStreamParser</span> <span class="pl-k">val</span> <span class="pl-smi">warehouse</span> <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-en">File</span>(<span class="pl-s"><span class="pl-pds">"</span>./warehouse<span class="pl-pds">"</span></span>).getCanonicalPath <span class="pl-k">val</span> <span class="pl-smi">metastore</span> <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-en">File</span>(<span class="pl-s"><span class="pl-pds">"</span>./metastore<span class="pl-pds">"</span></span>).getCanonicalPath @@ -241,6 +225,8 @@ .option(<span class="pl-s"><span class="pl-pds">"</span>checkpointLocation<span class="pl-pds">"</span></span>, <span class="pl-en">CarbonTablePath</span>.getStreamingCheckpointDir(tablePath)) .option(<span class="pl-s"><span class="pl-pds">"</span>dbName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>default<span class="pl-pds">"</span></span>) .option(<span class="pl-s"><span class="pl-pds">"</span>tableName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>carbon_table<span class="pl-pds">"</span></span>) + .option(<span class="pl-en">CarbonStreamParser</span>.<span class="pl-en">CARBON_STREAM_PARSER</span>, + <span class="pl-en">CarbonStreamParser</span>.<span class="pl-en">CARBON_STREAM_PARSER_CSV</span>) .start() <span class="pl-c"><span class="pl-c">//</span> start new thread to show data</span> @@ -370,14 +356,14 @@ streaming table using following DDL.</p> <tbody> <tr> <td>carbon.stream.parser</td> -<td>org.apache.carbondata.streaming.parser.CSVStreamParserImp</td> +<td>org.apache.carbondata.streaming.parser.RowStreamParserImp</td> <td>the class of the stream parser</td> </tr> </tbody> </table> <p>Currently CarbonData support two parsers, as following:</p> -<p><strong>1. org.apache.carbondata.streaming.parser.CSVStreamParserImp</strong>: This is the default stream parser, it gets a line data(String type) from the first index of InternalRow and converts this String to Object[].</p> -<p><strong>2. org.apache.carbondata.streaming.parser.RowStreamParserImp</strong>: This stream parser will auto convert InternalRow to Object[] according to schema of this <code>DataSet</code>, for example:</p> +<p><strong>1. org.apache.carbondata.streaming.parser.CSVStreamParserImp</strong>: This parser gets a line data(String type) from the first index of InternalRow and converts this String to Object[].</p> +<p><strong>2. org.apache.carbondata.streaming.parser.RowStreamParserImp</strong>: This is the default stream parser, it will auto convert InternalRow to Object[] according to schema of this <code>DataSet</code>, for example:</p> <div class="highlight highlight-source-scala"><pre> <span class="pl-k">case</span> <span class="pl-k">class</span> <span class="pl-en">FileElement</span>(<span class="pl-v">school</span>: <span class="pl-en">Array</span>[<span class="pl-k">String</span>], <span class="pl-v">age</span>: <span class="pl-k">Int</span>) <span class="pl-k">case</span> <span class="pl-k">class</span> <span class="pl-en">StreamData</span>(<span class="pl-v">id</span>: <span class="pl-k">Int</span>, <span class="pl-v">name</span>: <span class="pl-k">String</span>, <span class="pl-v">city</span>: <span class="pl-k">String</span>, <span class="pl-v">salary</span>: <span class="pl-k">Float</span>, <span class="pl-v">file</span>: <span class="pl-en">FileElement</span>) ... @@ -403,8 +389,6 @@ streaming table using following DDL.</p> .option(<span class="pl-s"><span class="pl-pds">"</span>checkpointLocation<span class="pl-pds">"</span></span>, tablePath.getStreamingCheckpointDir) .option(<span class="pl-s"><span class="pl-pds">"</span>dbName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>default<span class="pl-pds">"</span></span>) .option(<span class="pl-s"><span class="pl-pds">"</span>tableName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>carbon_table<span class="pl-pds">"</span></span>) - .option(<span class="pl-en">CarbonStreamParser</span>.<span class="pl-en">CARBON_STREAM_PARSER</span>, - <span class="pl-en">CarbonStreamParser</span>.<span class="pl-en">CARBON_STREAM_PARSER_ROW_PARSER</span>) .start() ...</pre></div> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/supported-data-types-in-carbondata.html ---------------------------------------------------------------------- diff --git a/content/supported-data-types-in-carbondata.html b/content/supported-data-types-in-carbondata.html index 0f06a42..780cab0 100644 --- a/content/supported-data-types-in-carbondata.html +++ b/content/supported-data-types-in-carbondata.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> @@ -206,6 +188,8 @@ <li>CHAR</li> <li>VARCHAR</li> </ul> +<p><strong>NOTE</strong>: For string longer than 32000 characters, use <code>LONG_STRING_COLUMNS</code> in table property. +Please refer to TBLProperties in <a href="https://github.com/apache/carbondata/blob/master/docs/data-management-on-carbondata.html#create-table";>CreateTable</a> for more information.</p> </li> <li> <p>Complex Types</p> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/timeseries-datamap-guide.html ---------------------------------------------------------------------- diff --git a/content/timeseries-datamap-guide.html b/content/timeseries-datamap-guide.html index 2400ce0..548be2e 100644 --- a/content/timeseries-datamap-guide.html +++ b/content/timeseries-datamap-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> @@ -176,7 +158,8 @@ <div id="viewpage" name="viewpage"> <div class="row"> <div class="col-sm-12 col-md-12"> - <div><h1> + <div> +<h1> <a id="carbondata-timeseries-datamap" class="anchor" href="#carbondata-timeseries-datamap" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Timeseries DataMap</h1> <ul> <li><a href="#timeseries-datamap-intoduction">Timeseries DataMap Introduction</a></li> @@ -184,7 +167,7 @@ <li><a href="#data-management-with-pre-aggregate-tables">Data Management</a></li> </ul> <h2> -<a id="timeseries-datamap-introduction-alpha-feature-in-130" class="anchor" href="#timeseries-datamap-introduction-alpha-feature-in-130" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Timeseries DataMap Introduction (Alpha feature in 1.3.0)</h2> +<a id="timeseries-datamap-introduction-alpha-feature" class="anchor" href="#timeseries-datamap-introduction-alpha-feature" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Timeseries DataMap Introduction (Alpha Feature)</h2> <p>Timeseries DataMap a pre-aggregate table implementation based on 'pre-aggregate' DataMap. Difference is that Timeseries DataMap has built-in understanding of time hierarchy and levels: year, month, day, hour, minute, so that it supports automatic roll-up in time dimension http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/troubleshooting.html ---------------------------------------------------------------------- diff --git a/content/troubleshooting.html b/content/troubleshooting.html index d5cf1b2..c668dc9 100644 --- a/content/troubleshooting.html +++ b/content/troubleshooting.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/content/useful-tips-on-carbondata.html ---------------------------------------------------------------------- diff --git a/content/useful-tips-on-carbondata.html b/content/useful-tips-on-carbondata.html index e8b0ab0..912cb48 100644 --- a/content/useful-tips-on-carbondata.html +++ b/content/useful-tips-on-carbondata.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul> @@ -180,7 +162,7 @@ <h1> <a id="useful-tips" class="anchor" href="#useful-tips" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Useful Tips</h1> <p>This tutorial guides you to create CarbonData Tables and optimize performance. -The following sections will elaborate on the above topics :</p> +The following sections will elaborate on the below topics :</p> <ul> <li><a href="#suggestions-to-create-carbondata-table">Suggestions to create CarbonData Table</a></li> <li><a href="#configuration-for-optimizing-data-loading-performance-for-massive-data">Configuration for Optimizing Data Loading performance for Massive Data</a></li> @@ -326,6 +308,10 @@ Incremental values are efficient in using min/max index. The create table comman )STORED BY 'carbondata' TBLPROPERTIES ('SORT_COLUMNS'='Dime_1, HOST, MSISDN') </code></pre> +<p><strong>NOTE:</strong></p> +<ul> +<li>BloomFilter can be created to enhance performance for queries with precise equal/in conditions. You can find more information about it in BloomFilter datamap <a href="https://github.com/apache/carbondata/blob/master/docs/datamap/bloomfilter-datamap-guide.html"; target=_blank>document</a>.</li> +</ul> <h2> <a id="configuration-for-optimizing-data-loading-performance-for-massive-data" class="anchor" href="#configuration-for-optimizing-data-loading-performance-for-massive-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Configuration for Optimizing Data Loading performance for Massive Data</h2> <p>CarbonData supports large data load, in this process sorting data while loading consumes a lot of memory and disk IO and @@ -457,7 +443,7 @@ scenarios. After the completion of POC, some of the configurations impacting the <td>spark/carbonlib/carbon.properties</td> <td>Data loading</td> <td>Specify the name of compressor to compress the intermediate sort temporary files during sort procedure in data loading.</td> -<td>The optional values are 'SNAPPY','GZIP','BZIP2','LZ4' and empty. By default, empty means that Carbondata will not compress the sort temp files. This parameter will be useful if you encounter disk bottleneck.</td> +<td>The optional values are 'SNAPPY','GZIP','BZIP2','LZ4','ZSTD' and empty. By default, empty means that Carbondata will not compress the sort temp files. This parameter will be useful if you encounter disk bottleneck.</td> </tr> <tr> <td>carbon.load.skewedDataOptimization.enabled</td> @@ -466,6 +452,13 @@ scenarios. After the completion of POC, some of the configurations impacting the <td>Whether to enable size based block allocation strategy for data loading.</td> <td>When loading, carbondata will use file size based block allocation strategy for task distribution. It will make sure that all the executors process the same size of data -- It's useful if the size of your input data files varies widely, say 1MB~1GB.</td> </tr> +<tr> +<td>carbon.load.min.size.enabled</td> +<td>spark/carbonlib/carbon.properties</td> +<td>Data loading</td> +<td>Whether to enable node minumun input data size allocation strategy for data loading.</td> +<td>When loading, carbondata will use node minumun input data size allocation strategy for task distribution. It will make sure the node load the minimum amount of data -- It's useful if the size of your input data files very small, say 1MB~256MB,Avoid generating a large number of small files.</td> +</tr> </tbody> </table> <p>Note: If your CarbonData instance is provided only for query, you may specify the property 'spark.speculation=true' which is in conf directory of spark.</p> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index df5280b..ba5b4b5 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -9,13 +9,15 @@ fileList=["configuration-parameters", "troubleshooting", "useful-tips-on-carbondata", "sdk-guide", + "s3-guide", "datamap-developer-guide" ] dataMapFileList=[ "bloomfilter-datamap-guide", "lucene-datamap-guide", "preaggregate-datamap-guide", - "timeseries-datamap-guide"] + "timeseries-datamap-guide", + "datamap-management"] dataMapFilesUrl="https://raw.githubusercontent.com/apache/carbondata/master/docs/datamap/"; http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/14d670df/src/main/scala/html/header.html ---------------------------------------------------------------------- diff --git a/src/main/scala/html/header.html b/src/main/scala/html/header.html index 470de01..9e904d0 100644 --- a/src/main/scala/html/header.html +++ b/src/main/scala/html/header.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; target="_blank">Apache CarbonData 1.4.0</a></li> <li> @@ -60,27 +63,6 @@ <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; target="_blank">Apache CarbonData 1.3.0</a></li> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; - target="_blank">Apache CarbonData 1.2.0</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Apache CarbonData 1.1.1</a></li> - <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; - target="_blank">Apache CarbonData 1.0.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; - target="_blank">Apache CarbonData 0.2.0</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; - target="_blank">Apache CarbonData 0.1.1</a></li> - <li> - <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; - target="_blank">Apache CarbonData 0.1.0</a></li> - <li> <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; target="_blank">Release Archive</a></li> </ul>
