Changes for release 1.3.0
Project: http://git-wip-us.apache.org/repos/asf/carbondata-site/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata-site/commit/4a4aad2b Tree: http://git-wip-us.apache.org/repos/asf/carbondata-site/tree/4a4aad2b Diff: http://git-wip-us.apache.org/repos/asf/carbondata-site/diff/4a4aad2b Branch: refs/heads/asf-site Commit: 4a4aad2bc61dd64feead7ca668bed0d739a27a40 Parents: 711502d Author: Jatin <jatin.de...@knoldus.in> Authored: Wed Feb 7 19:32:39 2018 +0530 Committer: Jatin <jatin.de...@knoldus.in> Committed: Wed Feb 7 20:53:34 2018 +0530 ---------------------------------------------------------------------- carbonscript.sh | 2 +- content/WEB-INF/classes/MDFileConverter.class | Bin 8964 -> 8964 bytes content/WEB-INF/classes/MdFileHandler.class | Bin 6144 -> 6144 bytes content/WEB-INF/classes/WebsiteLauncher$.class | Bin 2104 -> 4248 bytes content/WEB-INF/classes/WebsiteLauncher.class | Bin 713 -> 846 bytes content/WEB-INF/classes/application.conf | 1 + content/WEB-INF/classes/html/header.html | 3 + content/configuration-parameters.html | 3 + content/data-management-on-carbondata.html | 76 ++-- content/faq.html | 3 + content/file-structure-of-carbondata.html | 3 + content/index.html | 21 +- content/installation-guide.html | 3 + content/mainpage.html | 4 + content/pdf/maven-pdf-plugin.pdf | Bin 198959 -> 216771 bytes content/quick-start-guide.html | 3 + content/security.html | 3 + content/streaming-guide.html | 384 +++++++++++++++++++ content/supported-data-types-in-carbondata.html | 3 + content/troubleshooting.html | 3 + content/useful-tips-on-carbondata.html | 3 + content/videogallery.html | 4 + src/main/resources/application.conf | 1 + src/main/scala/MDFileConverter.scala | 1 + src/main/scala/WebsiteLauncher.scala | 12 + src/main/scala/html/header.html | 3 + src/main/webapp/configuration-parameters.html | 3 + .../webapp/data-management-on-carbondata.html | 76 ++-- src/main/webapp/faq.html | 3 + .../webapp/file-structure-of-carbondata.html | 3 + src/main/webapp/index.html | 21 +- src/main/webapp/installation-guide.html | 3 + src/main/webapp/mainpage.html | 4 + src/main/webapp/pdf/maven-pdf-plugin.pdf | Bin 198959 -> 216771 bytes src/main/webapp/quick-start-guide.html | 3 + src/main/webapp/security.html | 3 + src/main/webapp/streaming-guide.html | 384 +++++++++++++++++++ .../supported-data-types-in-carbondata.html | 3 + src/main/webapp/troubleshooting.html | 3 + src/main/webapp/useful-tips-on-carbondata.html | 3 + src/main/webapp/videogallery.html | 4 + .../markdown/data-management-on-carbondata.md | 72 ++-- src/site/markdown/streaming-guide.md | 169 ++++++++ src/site/pdf.xml | 3 +- 44 files changed, 1142 insertions(+), 157 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/carbonscript.sh ---------------------------------------------------------------------- diff --git a/carbonscript.sh b/carbonscript.sh index 31453e4..c57e933 100755 --- a/carbonscript.sh +++ b/carbonscript.sh @@ -6,4 +6,4 @@ mvn pdf:pdf mvn install -mvn exec:java -Dexec.mainClass=CleanUp \ No newline at end of file +# mvn exec:java -Dexec.mainClass=CleanUp \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/WEB-INF/classes/MDFileConverter.class ---------------------------------------------------------------------- diff --git a/content/WEB-INF/classes/MDFileConverter.class b/content/WEB-INF/classes/MDFileConverter.class index 84b27b6..7b74cf9 100644 Binary files a/content/WEB-INF/classes/MDFileConverter.class and b/content/WEB-INF/classes/MDFileConverter.class differ http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/WEB-INF/classes/MdFileHandler.class ---------------------------------------------------------------------- diff --git a/content/WEB-INF/classes/MdFileHandler.class b/content/WEB-INF/classes/MdFileHandler.class index 58088b3..f39c098 100644 Binary files a/content/WEB-INF/classes/MdFileHandler.class and b/content/WEB-INF/classes/MdFileHandler.class differ http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/WEB-INF/classes/WebsiteLauncher$.class ---------------------------------------------------------------------- diff --git a/content/WEB-INF/classes/WebsiteLauncher$.class b/content/WEB-INF/classes/WebsiteLauncher$.class index fa141ff..9e36ad5 100644 Binary files a/content/WEB-INF/classes/WebsiteLauncher$.class and b/content/WEB-INF/classes/WebsiteLauncher$.class differ http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/WEB-INF/classes/WebsiteLauncher.class ---------------------------------------------------------------------- diff --git a/content/WEB-INF/classes/WebsiteLauncher.class b/content/WEB-INF/classes/WebsiteLauncher.class index 52163dd..82fab5d 100644 Binary files a/content/WEB-INF/classes/WebsiteLauncher.class and b/content/WEB-INF/classes/WebsiteLauncher.class differ http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/WEB-INF/classes/application.conf ---------------------------------------------------------------------- diff --git a/content/WEB-INF/classes/application.conf b/content/WEB-INF/classes/application.conf index cfffec0..e20ebe4 100644 --- a/content/WEB-INF/classes/application.conf +++ b/content/WEB-INF/classes/application.conf @@ -4,6 +4,7 @@ fileList=["configuration-parameters", "faq", "file-structure-of-carbondata", "installation-guide", + "streaming-guide", "supported-data-types-in-carbondata", "troubleshooting", "useful-tips-on-carbondata" http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/WEB-INF/classes/html/header.html ---------------------------------------------------------------------- diff --git a/content/WEB-INF/classes/html/header.html b/content/WEB-INF/classes/html/header.html index 39e5fda..5ce0b5d 100644 --- a/content/WEB-INF/classes/html/header.html +++ b/content/WEB-INF/classes/html/header.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/configuration-parameters.html ---------------------------------------------------------------------- diff --git a/content/configuration-parameters.html b/content/configuration-parameters.html index 4d21876..49b580f 100644 --- a/content/configuration-parameters.html +++ b/content/configuration-parameters.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/data-management-on-carbondata.html ---------------------------------------------------------------------- diff --git a/content/data-management-on-carbondata.html b/content/data-management-on-carbondata.html index ece2f04..846f11e 100644 --- a/content/data-management-on-carbondata.html +++ b/content/data-management-on-carbondata.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> @@ -663,30 +666,29 @@ STORED BY 'carbondata' <a id="load-data-using-static-partition" class="anchor" href="#load-data-using-static-partition" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Load Data Using Static Partition</h4> <p>This command allows you to load data using static partition.</p> <pre><code>LOAD DATA [LOCAL] INPATH 'folder_path' - INTO TABLE [db_name.]table_name PARTITION (partition_spec) - OPTIONS(property_name=property_value, ...) -NSERT INTO INTO TABLE [db_name.]table_name PARTITION (partition_spec) SELECT STATMENT +INTO TABLE [db_name.]table_name PARTITION (partition_spec) +OPTIONS(property_name=property_value, ...) + +INSERT INTO INTO TABLE [db_name.]table_name PARTITION (partition_spec) <SELECT STATMENT> </code></pre> <p>Example:</p> -<pre><code>LOAD DATA LOCAL INPATH '${env:HOME}/staticinput.txt' - INTO TABLE locationTable - PARTITION (country = 'US', state = 'CA') +<pre><code>LOAD DATA LOCAL INPATH '${env:HOME}/staticinput.csv' +INTO TABLE locationTable +PARTITION (country = 'US', state = 'CA') INSERT INTO TABLE locationTable - PARTITION (country = 'US', state = 'AL') - SELECT * FROM another_user au - WHERE au.country = 'US' AND au.state = 'AL'; +PARTITION (country = 'US', state = 'AL') +SELECT <columns list excluding partition columns> FROM another_user </code></pre> <h4> <a id="load-data-using-dynamic-partition" class="anchor" href="#load-data-using-dynamic-partition" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Load Data Using Dynamic Partition</h4> <p>This command allows you to load data using dynamic partition. If partition spec is not specified, then the partition is considered as dynamic.</p> <p>Example:</p> -<pre><code>LOAD DATA LOCAL INPATH '${env:HOME}/staticinput.txt' - INTO TABLE locationTable +<pre><code>LOAD DATA LOCAL INPATH '${env:HOME}/staticinput.csv' +INTO TABLE locationTable INSERT INTO TABLE locationTable - SELECT * FROM another_user au - WHERE au.country = 'US' AND au.state = 'AL'; +SELECT <columns list excluding partition columns> FROM another_user </code></pre> <h4> <a id="show-partitions" class="anchor" href="#show-partitions" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Show Partitions</h4> @@ -702,17 +704,17 @@ INSERT INTO TABLE locationTable <a id="insert-overwrite" class="anchor" href="#insert-overwrite" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Insert OVERWRITE</h4> <p>This command allows you to insert or load overwrite on a spcific partition.</p> <pre><code> INSERT OVERWRITE TABLE table_name - PARTITION (column = 'partition_name') - select_statement + PARTITION (column = 'partition_name') + select_statement </code></pre> <p>Example:</p> <pre><code>INSERT OVERWRITE TABLE partitioned_user - PARTITION (country = 'US') - SELECT * FROM another_user au - WHERE au.country = 'US'; +PARTITION (country = 'US') +SELECT * FROM another_user au +WHERE au.country = 'US'; </code></pre> <h3> -<a id="carbondata-partitionhashrangelist----alpha-feature-this-partition-not-supports-update-and-delete-data" class="anchor" href="#carbondata-partitionhashrangelist----alpha-feature-this-partition-not-supports-update-and-delete-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CARBONDATA PARTITION(HASH,RANGE,LIST) -- Alpha feature, this partition not supports update and delete data.</h3> +<a id="carbondata-partitionhashrangelist----alpha-feature-this-partition-feature-does-not-support-update-and-delete-data" class="anchor" href="#carbondata-partitionhashrangelist----alpha-feature-this-partition-feature-does-not-support-update-and-delete-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CARBONDATA PARTITION(HASH,RANGE,LIST) -- Alpha feature, this partition feature does not support update and delete data.</h3> <p>The partition supports three type:(Hash,Range,List), similar to other system's partition features, CarbonData's partition feature can be used to improve query performance by filtering on the partition column.</p> <h3> <a id="create-hash-partition-table" class="anchor" href="#create-hash-partition-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create Hash Partition Table</h3> @@ -890,11 +892,11 @@ SELECT sum(price), country from sales GROUP BY country </code></pre> <p>will be transformed by Query Planner to fetch data from pre-aggregate table <strong>agg_sales</strong></p> <p>But queries of kind</p> -<pre><code>SELECT user_id, country, sex, sum(quantity), avg(price) from sales GROUP BY country, sex +<pre><code>SELECT user_id, country, sex, sum(quantity), avg(price) from sales GROUP BY user_id, country, sex SELECT sex, avg(quantity) from sales GROUP BY sex -SELECT max(price), country from sales GROUP BY country +SELECT country, max(price) from sales GROUP BY country </code></pre> <p>will fetch the data from the main table <strong>sales</strong></p> <h5> @@ -913,18 +915,12 @@ pre-aggregate tables satisfy the query condition, the plan is transformed automa pre-aggregate table to fetch the data</p> <h5> <a id="compacting-pre-aggregate-tables" class="anchor" href="#compacting-pre-aggregate-tables" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compacting pre-aggregate tables</h5> -<p>Compaction is an optional operation for pre-aggregate table. If compaction is performed on main -table but not performed on pre-aggregate table, all queries still can benefit from pre-aggregate -table.To further improve performance on pre-aggregate table, compaction can be triggered on -pre-aggregate tables directly, it will merge the segments inside pre-aggregation table. -To do that, use ALTER TABLE COMPACT command on the pre-aggregate table just like the main table</p> -<p>NOTE:</p> -<ul> -<li>If the aggregate function used in the pre-aggregate table creation included distinct-count, -during compaction, the pre-aggregate table values are recomputed.This would a costly -operation as compared to the compaction of pre-aggregate tables containing other aggregate -functions alone</li> -</ul> +<p>Compaction command (ALTER TABLE COMPACT) need to be run separately on each pre-aggregate table. +Running Compaction command on main table will <strong>not automatically</strong> compact the pre-aggregate +tables.Compaction is an optional operation for pre-aggregate table. If compaction is performed on +main table but not performed on pre-aggregate table, all queries still can benefit from +pre-aggregate tables.To further improve performance on pre-aggregate tables, compaction can be +triggered on pre-aggregate tables directly, it will merge the segments inside pre-aggregate table.</p> <h5> <a id="updatedelete-operations-on-pre-aggregate-tables" class="anchor" href="#updatedelete-operations-on-pre-aggregate-tables" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Update/Delete Operations on pre-aggregate tables</h5> <p>This functionality is not supported.</p> @@ -1010,16 +1006,6 @@ DMPROPERTIES ( ) AS SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price), avg(price) FROM sales GROUP BY order_time, country, sex - -CREATE DATAMAP agg_minute -ON TABLE sales -USING "timeseries" -DMPROPERTIES ( -'event_time?=?order_time?, -'minute_granualrity?=?1?, -) AS -SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price), - avg(price) FROM sales GROUP BY order_time, country, sex </code></pre> <p>For Querying data and automatically roll-up to the desired aggregation level,Carbondata supports UDF as</p> @@ -1029,9 +1015,7 @@ UDF as</p> ?hour?) </code></pre> <p>It is <strong>not necessary</strong> to create pre-aggregate tables for each granularity unless required for -query -.Carbondata -can roll-up the data and fetch it</p> +query.Carbondata can roll-up the data and fetch it.</p> <p>For Example: For main table <strong>sales</strong> , If pre-aggregate tables were created as</p> <pre><code>CREATE DATAMAP agg_day ON TABLE sales http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/faq.html ---------------------------------------------------------------------- diff --git a/content/faq.html b/content/faq.html index 0423240..b42f8bd 100644 --- a/content/faq.html +++ b/content/faq.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/file-structure-of-carbondata.html ---------------------------------------------------------------------- diff --git a/content/file-structure-of-carbondata.html b/content/file-structure-of-carbondata.html index 2db223a..1c8e758 100644 --- a/content/file-structure-of-carbondata.html +++ b/content/file-structure-of-carbondata.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/index.html ---------------------------------------------------------------------- diff --git a/content/index.html b/content/index.html index 40ab6b5..f2cbbbf 100644 --- a/content/index.html +++ b/content/index.html @@ -54,6 +54,9 @@ class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> @@ -319,6 +322,13 @@ </h4> <div class="linkblock"> <div class="block-row"> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a> + <span class="release-date">Feb 2018</span> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.3.0+Release"; + class="whatsnew" target="_blank">what's new</a> + </div> + <div class="block-row"> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a> <span class="release-date">Sept 2017</span> @@ -332,13 +342,6 @@ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.1.1+Release"; class="whatsnew" target="_blank">what's new</a> </div> - <div class="block-row"> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a> - <span class="release-date">May 2017</span> - <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.1.0+Release"; - class="whatsnew" target="_blank">what's new</a> - </div> </div> </div> <div class="nextR"> @@ -475,8 +478,8 @@ to do is:</p> <ol class="orderlist"> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Download </a>the latest release. + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Download</a>the latest release. </li> <li>Follow the <a href="quick-start-guide.html">Quick Start</a>. http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/installation-guide.html ---------------------------------------------------------------------- diff --git a/content/installation-guide.html b/content/installation-guide.html index d8c0413..7da254e 100644 --- a/content/installation-guide.html +++ b/content/installation-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/mainpage.html ---------------------------------------------------------------------- diff --git a/content/mainpage.html b/content/mainpage.html index 52ffdb4..d782c46 100644 --- a/content/mainpage.html +++ b/content/mainpage.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> @@ -191,6 +194,7 @@ <li><a href="installation-guide.html">Installation Guide</a></li> <li><a href="configuration-parameters.html">Configuring CarbonData</a></li> + <li><a href="streaming-guide.html">Streaming Guide</a></li> <li><a href="faq.html">FAQs</a></li> <li><a href="troubleshooting.html">Troubleshooting</a></li> <li><a href="useful-tips-on-carbondata.html">Useful http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/pdf/maven-pdf-plugin.pdf ---------------------------------------------------------------------- diff --git a/content/pdf/maven-pdf-plugin.pdf b/content/pdf/maven-pdf-plugin.pdf index b37099a..72c0425 100644 Binary files a/content/pdf/maven-pdf-plugin.pdf and b/content/pdf/maven-pdf-plugin.pdf differ http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/quick-start-guide.html ---------------------------------------------------------------------- diff --git a/content/quick-start-guide.html b/content/quick-start-guide.html index eccdabb..71ca760 100644 --- a/content/quick-start-guide.html +++ b/content/quick-start-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/security.html ---------------------------------------------------------------------- diff --git a/content/security.html b/content/security.html index 56ec55e..3822acb 100644 --- a/content/security.html +++ b/content/security.html @@ -45,6 +45,9 @@ aria-expanded="false">Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/streaming-guide.html ---------------------------------------------------------------------- diff --git a/content/streaming-guide.html b/content/streaming-guide.html new file mode 100644 index 0000000..8d3effe --- /dev/null +++ b/content/streaming-guide.html @@ -0,0 +1,384 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>CarbonData</title> + <style> + + </style> + <!-- Bootstrap --> + + <link rel="stylesheet" href="css/bootstrap.min.css"> + <link href="css/style.css" rel="stylesheet"> + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js";></script> + <![endif]--> + <script src="js/jquery.min.js"></script> + <script src="js/bootstrap.min.js"></script> + + +</head> +<body> +<header> + <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper"> + <div class="container"> + <div class="navbar-header"> + <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse" + class="navbar-toggle collapsed" type="button"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a href="index.html" class="logo"> + <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/> + </a> + </div> + <div class="navbar-collapse collapse cd_navcontnt" id="navbar"> + <ul class="nav navbar-nav navbar-right navlist-custom"> + <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a> + </li> + <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false"> Download <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; + target="_blank">Apache CarbonData 1.2.0</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; + target="_blank">Apache CarbonData 1.1.1</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; + target="_blank">Apache CarbonData 1.1.0</a></li> + <li> + <a href="http://archive.apache.org/dist/incubator/carbondata/1.0.0-incubating/"; + target="_blank">Apache CarbonData 1.0.0</a></li> + <li> + <a href="http://archive.apache.org/dist/incubator/carbondata/0.2.0-incubating/"; + target="_blank">Apache CarbonData 0.2.0</a></li> + <li> + <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.1-incubating/"; + target="_blank">Apache CarbonData 0.1.1</a></li> + <li> + <a href="http://archive.apache.org/dist/incubator/carbondata/0.1.0-incubating/"; + target="_blank">Apache CarbonData 0.1.0</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; + target="_blank">Release Archive</a></li> + </ul> + </li> + <li><a href="mainpage.html" class="active">Documentation</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false">Community <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"; + target="_blank">Contributing to CarbonData</a></li> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"; + target="_blank">Release Guide</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"; + target="_blank">Project PMC and Committers</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"; + target="_blank">CarbonData Meetups</a></li> + <li><a href="security.html">Apache CarbonData Security</a></li> + <li><a href="https://issues.apache.org/jira/browse/CARBONDATA"; target="_blank">Apache + Jira</a></li> + <li><a href="videogallery.html">CarbonData Videos </a></li> + </ul> + </li> + <li class="dropdown"> + <a href="http://www.apache.org/"; class="apache_link hidden-xs dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li class="dropdown"> + <a href="http://www.apache.org/"; class="hidden-lg hidden-md hidden-sm dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li> + <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a> + + </li> + + </ul> + </div><!--/.nav-collapse --> + <div id="search-box"> + <form method="get" action="http://www.google.com/search"; target="_blank"> + <div class="search-block"> + <table border="0" cellpadding="0" width="100%"> + <tr> + <td style="width:80%"> + <input type="text" name="q" size=" 5" maxlength="255" value="" + class="search-input" placeholder="Search...." required/> + </td> + <td style="width:20%"> + <input type="submit" value="Search"/></td> + </tr> + <tr> + <td align="left" style="font-size:75%" colspan="2"> + <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/> + <span style=" position: relative; top: -3px;"> Only search for CarbonData</span> + </td> + </tr> + </table> + </div> + </form> + </div> + </div> + </nav> +</header> <!-- end Header part --> + +<div class="fixed-padding"></div> <!-- top padding with fixde header --> + +<section><!-- Dashboard nav --> + <div class="container-fluid q"> + <div class="col-sm-12 col-md-12 maindashboard"> + <div class="row"> + <section> + <div style="padding:10px 15px;"> + <div id="viewpage" name="viewpage"> + <div class="row"> + <div class="col-sm-12 col-md-12"> + <div><h1> +<a id="carbondata-streaming-ingestion" class="anchor" href="#carbondata-streaming-ingestion" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData Streaming Ingestion</h1> +<h2> +<a id="quick-example" class="anchor" href="#quick-example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Quick example</h2> +<p>Download and unzip spark-2.2.0-bin-hadoop2.7.tgz, and export $SPARK_HOME</p> +<p>Package carbon jar, and copy assembly/target/scala-2.11/carbondata_2.11-1.3.0-SNAPSHOT-shade-hadoop2.7.2.jar to $SPARK_HOME/jars</p> +<div class="highlight highlight-source-shell"><pre>mvn clean package -DskipTests -Pspark-2.2</pre></div> +<p>Start a socket data server in a terminal</p> +<div class="highlight highlight-source-shell"><pre> nc -lk 9099</pre></div> +<p>type some CSV rows as following</p> +<pre lang="csv"><code>1,col1 +2,col2 +3,col3 +4,col4 +5,col5 +</code></pre> +<p>Start spark-shell in new terminal, type :paste, then copy and run the following code.</p> +<div class="highlight highlight-source-scala"><pre> <span class="pl-k">import</span> <span class="pl-smi">java.io.</span><span class="pl-smi">File</span> + <span class="pl-k">import</span> <span class="pl-smi">org.apache.spark.sql.</span>{<span class="pl-smi">CarbonEnv</span>, <span class="pl-smi">SparkSession</span>} + <span class="pl-k">import</span> <span class="pl-smi">org.apache.spark.sql.CarbonSession.</span><span class="pl-smi">_</span> + <span class="pl-k">import</span> <span class="pl-smi">org.apache.spark.sql.streaming.</span>{<span class="pl-smi">ProcessingTime</span>, <span class="pl-smi">StreamingQuery</span>} + <span class="pl-k">import</span> <span class="pl-smi">org.apache.carbondata.core.util.path.</span><span class="pl-smi">CarbonStorePath</span> + + <span class="pl-k">val</span> <span class="pl-en">warehouse</span> <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-en">File</span>(<span class="pl-s"><span class="pl-pds">"</span>./warehouse<span class="pl-pds">"</span></span>).getCanonicalPath + <span class="pl-k">val</span> <span class="pl-en">metastore</span> <span class="pl-k">=</span> <span class="pl-k">new</span> <span class="pl-en">File</span>(<span class="pl-s"><span class="pl-pds">"</span>./metastore<span class="pl-pds">"</span></span>).getCanonicalPath + + <span class="pl-k">val</span> <span class="pl-en">spark</span> <span class="pl-k">=</span> <span class="pl-en">SparkSession</span> + .builder() + .master(<span class="pl-s"><span class="pl-pds">"</span>local<span class="pl-pds">"</span></span>) + .appName(<span class="pl-s"><span class="pl-pds">"</span>StreamExample<span class="pl-pds">"</span></span>) + .config(<span class="pl-s"><span class="pl-pds">"</span>spark.sql.warehouse.dir<span class="pl-pds">"</span></span>, warehouse) + .getOrCreateCarbonSession(warehouse, metastore) + + spark.sparkContext.setLogLevel(<span class="pl-s"><span class="pl-pds">"</span>ERROR<span class="pl-pds">"</span></span>) + + <span class="pl-c"><span class="pl-c">//</span> drop table if exists previously</span> + spark.sql(s<span class="pl-s"><span class="pl-pds">"</span>DROP TABLE IF EXISTS carbon_table<span class="pl-pds">"</span></span>) + <span class="pl-c"><span class="pl-c">//</span> Create target carbon table and populate with initial data</span> + spark.sql( + s<span class="pl-s"><span class="pl-pds">"""</span></span> +<span class="pl-s"> | CREATE TABLE carbon_table (</span> +<span class="pl-s"> | col1 INT,</span> +<span class="pl-s"> | col2 STRING</span> +<span class="pl-s"> | )</span> +<span class="pl-s"> | STORED BY 'carbondata'</span> +<span class="pl-s"> | TBLPROPERTIES('streaming'='true')<span class="pl-pds">"""</span></span>.stripMargin) + + <span class="pl-k">val</span> <span class="pl-en">carbonTable</span> <span class="pl-k">=</span> <span class="pl-en">CarbonEnv</span>.getCarbonTable(<span class="pl-en">Some</span>(<span class="pl-s"><span class="pl-pds">"</span>default<span class="pl-pds">"</span></span>), <span class="pl-s"><span class="pl-pds">"</span>carbon_table<span class="pl-pds">"</span></span>)(spark) + <span class="pl-k">val</span> <span class="pl-en">tablePath</span> <span class="pl-k">=</span> <span class="pl-en">CarbonStorePath</span>.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier) + + <span class="pl-c"><span class="pl-c">//</span> batch load</span> + <span class="pl-k">var</span> <span class="pl-en">qry</span><span class="pl-k">:</span> <span class="pl-en">StreamingQuery</span> <span class="pl-k">=</span> <span class="pl-c1">null</span> + <span class="pl-k">val</span> <span class="pl-en">readSocketDF</span> <span class="pl-k">=</span> spark.readStream + .format(<span class="pl-s"><span class="pl-pds">"</span>socket<span class="pl-pds">"</span></span>) + .option(<span class="pl-s"><span class="pl-pds">"</span>host<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>localhost<span class="pl-pds">"</span></span>) + .option(<span class="pl-s"><span class="pl-pds">"</span>port<span class="pl-pds">"</span></span>, <span class="pl-c1">9099</span>) + .load() + + <span class="pl-c"><span class="pl-c">//</span> Write data from socket stream to carbondata file</span> + qry <span class="pl-k">=</span> readSocketDF.writeStream + .format(<span class="pl-s"><span class="pl-pds">"</span>carbondata<span class="pl-pds">"</span></span>) + .trigger(<span class="pl-en">ProcessingTime</span>(<span class="pl-s"><span class="pl-pds">"</span>5 seconds<span class="pl-pds">"</span></span>)) + .option(<span class="pl-s"><span class="pl-pds">"</span>checkpointLocation<span class="pl-pds">"</span></span>, tablePath.getStreamingCheckpointDir) + .option(<span class="pl-s"><span class="pl-pds">"</span>dbName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>default<span class="pl-pds">"</span></span>) + .option(<span class="pl-s"><span class="pl-pds">"</span>tableName<span class="pl-pds">"</span></span>, <span class="pl-s"><span class="pl-pds">"</span>carbon_table<span class="pl-pds">"</span></span>) + .start() + + <span class="pl-c"><span class="pl-c">//</span> start new thread to show data</span> + <span class="pl-k">new</span> <span class="pl-en">Thread</span>() { + <span class="pl-k">override</span> <span class="pl-k">def</span> <span class="pl-en">run</span>()<span class="pl-k">:</span> <span class="pl-k">Unit</span> <span class="pl-k">=</span> { + <span class="pl-k">do</span> { + spark.sql(<span class="pl-s"><span class="pl-pds">"</span>select * from carbon_table<span class="pl-pds">"</span></span>).show(<span class="pl-c1">false</span>) + <span class="pl-en">Thread</span>.sleep(<span class="pl-c1">10000</span>) + } <span class="pl-k">while</span> (<span class="pl-c1">true</span>) + } + }.start() + + qry.awaitTermination()</pre></div> +<p>Continue to type some rows into data server, and spark-shell will show the new data of the table.</p> +<h2> +<a id="create-table-with-streaming-property" class="anchor" href="#create-table-with-streaming-property" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create table with streaming property</h2> +<p>Streaming table is just a normal carbon table with "streaming" table property, user can create +streaming table using following DDL.</p> +<div class="highlight highlight-source-sql"><pre> <span class="pl-k">CREATE</span> <span class="pl-k">TABLE</span> <span class="pl-en">streaming_table</span> ( + col1 <span class="pl-k">INT</span>, + col2 STRING + ) + STORED BY <span class="pl-s"><span class="pl-pds">'</span>carbondata<span class="pl-pds">'</span></span> + TBLPROPERTIES(<span class="pl-s"><span class="pl-pds">'</span>streaming<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>true<span class="pl-pds">'</span></span>)</pre></div> +<table> +<thead> +<tr> +<th>property name</th> +<th>default</th> +<th>description</th> +</tr> +</thead> +<tbody> +<tr> +<td>streaming</td> +<td>false</td> +<td>Whether to enable streaming ingest feature for this table <br> Value range: true, false</td> +</tr> +</tbody> +</table> +<p>"DESC FORMATTED" command will show streaming property.</p> +<div class="highlight highlight-source-sql"><pre><span class="pl-k">DESC</span> FORMATTED streaming_table</pre></div> +<h2> +<a id="alter-streaming-property" class="anchor" href="#alter-streaming-property" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Alter streaming property</h2> +<p>For an old table, use ALTER TABLE command to set the streaming property.</p> +<div class="highlight highlight-source-sql"><pre><span class="pl-k">ALTER</span> <span class="pl-k">TABLE</span> streaming_table <span class="pl-k">SET</span> TBLPROPERTIES(<span class="pl-s"><span class="pl-pds">'</span>streaming<span class="pl-pds">'</span></span><span class="pl-k">=</span><span class="pl-s"><span class="pl-pds">'</span>true<span class="pl-pds">'</span></span>)</pre></div> +<h2> +<a id="acquire-streaming-lock" class="anchor" href="#acquire-streaming-lock" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Acquire streaming lock</h2> +<p>At the begin of streaming ingestion, the system will try to acquire the table level lock of streaming.lock file. If the system isn't able to acquire the lock of this table, it will throw an InterruptedException.</p> +<h2> +<a id="create-streaming-segment" class="anchor" href="#create-streaming-segment" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create streaming segment</h2> +<p>The input data of streaming will be ingested into a segment of the CarbonData table, the status of this segment is streaming. CarbonData call it a streaming segment. The "tablestatus" file will record the segment status and data size. The user can use ?SHOW SEGMENTS FOR TABLE tableName? to check segment status.</p> +<p>After the streaming segment reaches the max size, CarbonData will change the segment status to "streaming finish" from "streaming", and create new "streaming" segment to continue to ingest streaming data.</p> +<table> +<thead> +<tr> +<th>option</th> +<th>default</th> +<th>description</th> +</tr> +</thead> +<tbody> +<tr> +<td>carbon.streaming.segment.max.size</td> +<td>1024000000</td> +<td>Unit: byte <br>max size of streaming segment</td> +</tr> +</tbody> +</table> +<table> +<thead> +<tr> +<th>segment status</th> +<th>description</th> +</tr> +</thead> +<tbody> +<tr> +<td>streaming</td> +<td>The segment is running streaming ingestion</td> +</tr> +<tr> +<td>streaming finish</td> +<td>The segment already finished streaming ingestion, <br> it will be handed off to a segment in the columnar format</td> +</tr> +</tbody> +</table> +<h2> +<a id="change-segment-status" class="anchor" href="#change-segment-status" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Change segment status</h2> +<p>Use below command to change the status of "streaming" segment to "streaming finish" segment.</p> +<div class="highlight highlight-source-sql"><pre><span class="pl-k">ALTER</span> <span class="pl-k">TABLE</span> streaming_table FINISH STREAMING</pre></div> +<h2> +<a id="handoff-streaming-finish-segment-to-columnar-segment" class="anchor" href="#handoff-streaming-finish-segment-to-columnar-segment" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Handoff "streaming finish" segment to columnar segment</h2> +<p>Use below command to handoff "streaming finish" segment to columnar format segment manually.</p> +<div class="highlight highlight-source-sql"><pre><span class="pl-k">ALTER</span> <span class="pl-k">TABLE</span> streaming_table COMPACT <span class="pl-s"><span class="pl-pds">'</span>streaming<span class="pl-pds">'</span></span> +</pre></div> +<h2> +<a id="auto-handoff-streaming-segment" class="anchor" href="#auto-handoff-streaming-segment" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Auto handoff streaming segment</h2> +<p>Config the property "carbon.streaming.auto.handoff.enabled" to auto handoff streaming segment. If the value of this property is true, after the streaming segment reaches the max size, CarbonData will change this segment to "streaming finish" status and trigger to auto handoff this segment to columnar format segment in a new thread.</p> +<table> +<thead> +<tr> +<th>property name</th> +<th>default</th> +<th>description</th> +</tr> +</thead> +<tbody> +<tr> +<td>carbon.streaming.auto.handoff.enabled</td> +<td>true</td> +<td>whether to auto trigger handoff operation</td> +</tr> +</tbody> +</table> +<h2> +<a id="close-streaming-table" class="anchor" href="#close-streaming-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Close streaming table</h2> +<p>Use below command to handoff all streaming segments to columnar format segments and modify the streaming property to false, this table becomes a normal table.</p> +<div class="highlight highlight-source-sql"><pre><span class="pl-k">ALTER</span> <span class="pl-k">TABLE</span> streaming_table COMPACT <span class="pl-s"><span class="pl-pds">'</span>close_streaming<span class="pl-pds">'</span></span> +</pre></div> +<h2> +<a id="constraint" class="anchor" href="#constraint" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Constraint</h2> +<ol> +<li>reject set streaming property from true to false.</li> +<li>reject UPDATE/DELETE command on the streaming table.</li> +<li>reject create pre-aggregation DataMap on the streaming table.</li> +<li>reject add the streaming property on the table with pre-aggregation DataMap.</li> +<li>if the table has dictionary columns, it will not support concurrent data loading.</li> +<li>block delete "streaming" segment while the streaming ingestion is running.</li> +<li>block drop the streaming table while the streaming ingestion is running.</li> +</ol> +</div> +</div> +</div> +</div> +<div class="doc-footer"> + <a href="#top" class="scroll-top">Top</a> +</div> +</div> +</section> +</div> +</div> +</div> +</section><!-- End systemblock part --> +<script src="js/custom.js"></script> +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/supported-data-types-in-carbondata.html ---------------------------------------------------------------------- diff --git a/content/supported-data-types-in-carbondata.html b/content/supported-data-types-in-carbondata.html index 51a1216..dd4cbc5 100644 --- a/content/supported-data-types-in-carbondata.html +++ b/content/supported-data-types-in-carbondata.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/troubleshooting.html ---------------------------------------------------------------------- diff --git a/content/troubleshooting.html b/content/troubleshooting.html index 6129686..3a2e311 100644 --- a/content/troubleshooting.html +++ b/content/troubleshooting.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/useful-tips-on-carbondata.html ---------------------------------------------------------------------- diff --git a/content/useful-tips-on-carbondata.html b/content/useful-tips-on-carbondata.html index 3d36da8..cb19036 100644 --- a/content/useful-tips-on-carbondata.html +++ b/content/useful-tips-on-carbondata.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/content/videogallery.html ---------------------------------------------------------------------- diff --git a/content/videogallery.html b/content/videogallery.html index 154aad2..bfa5a49 100644 --- a/content/videogallery.html +++ b/content/videogallery.html @@ -49,6 +49,10 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index cfffec0..e20ebe4 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -4,6 +4,7 @@ fileList=["configuration-parameters", "faq", "file-structure-of-carbondata", "installation-guide", + "streaming-guide", "supported-data-types-in-carbondata", "troubleshooting", "useful-tips-on-carbondata" http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/scala/MDFileConverter.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/MDFileConverter.scala b/src/main/scala/MDFileConverter.scala index 1fd0bef..59cb7ad 100644 --- a/src/main/scala/MDFileConverter.scala +++ b/src/main/scala/MDFileConverter.scala @@ -1,6 +1,7 @@ import com.google.inject.Inject import org.slf4j.{Logger, LoggerFactory} import services.{ConfService, DataService, FileService} +import java.nio.file.{Files, Path, Paths} import scala.util.matching.Regex http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/scala/WebsiteLauncher.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/WebsiteLauncher.scala b/src/main/scala/WebsiteLauncher.scala index f814769..060796a 100644 --- a/src/main/scala/WebsiteLauncher.scala +++ b/src/main/scala/WebsiteLauncher.scala @@ -1,4 +1,5 @@ import java.io.File +import java.nio.file.{Files, Paths} import org.slf4j.{Logger, LoggerFactory} import services.{ConfServiceImpl, DataServiceImpl, FileServiceImpl} @@ -9,6 +10,10 @@ object WebsiteLauncher { val conf = new ConfServiceImpl + val mdFileLocation = conf.readString("mdFileLocation") + + deleteRecursively(new File(mdFileLocation)) + val file = init(conf.readString("mdFileLocation")) val dirStatus = file.mkdir() @@ -23,4 +28,11 @@ object WebsiteLauncher { new File(path) } + def deleteRecursively(file: File): Unit = { + if (file.isDirectory) + file.listFiles.foreach(deleteRecursively) + if (file.exists && !file.delete) + throw new Exception(s"Unable to delete ${file.getAbsolutePath}") + } + } http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/scala/html/header.html ---------------------------------------------------------------------- diff --git a/src/main/scala/html/header.html b/src/main/scala/html/header.html index 39e5fda..5ce0b5d 100644 --- a/src/main/scala/html/header.html +++ b/src/main/scala/html/header.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/configuration-parameters.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/configuration-parameters.html b/src/main/webapp/configuration-parameters.html index 4d21876..49b580f 100644 --- a/src/main/webapp/configuration-parameters.html +++ b/src/main/webapp/configuration-parameters.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/data-management-on-carbondata.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/data-management-on-carbondata.html b/src/main/webapp/data-management-on-carbondata.html index ece2f04..846f11e 100644 --- a/src/main/webapp/data-management-on-carbondata.html +++ b/src/main/webapp/data-management-on-carbondata.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> @@ -663,30 +666,29 @@ STORED BY 'carbondata' <a id="load-data-using-static-partition" class="anchor" href="#load-data-using-static-partition" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Load Data Using Static Partition</h4> <p>This command allows you to load data using static partition.</p> <pre><code>LOAD DATA [LOCAL] INPATH 'folder_path' - INTO TABLE [db_name.]table_name PARTITION (partition_spec) - OPTIONS(property_name=property_value, ...) -NSERT INTO INTO TABLE [db_name.]table_name PARTITION (partition_spec) SELECT STATMENT +INTO TABLE [db_name.]table_name PARTITION (partition_spec) +OPTIONS(property_name=property_value, ...) + +INSERT INTO INTO TABLE [db_name.]table_name PARTITION (partition_spec) <SELECT STATMENT> </code></pre> <p>Example:</p> -<pre><code>LOAD DATA LOCAL INPATH '${env:HOME}/staticinput.txt' - INTO TABLE locationTable - PARTITION (country = 'US', state = 'CA') +<pre><code>LOAD DATA LOCAL INPATH '${env:HOME}/staticinput.csv' +INTO TABLE locationTable +PARTITION (country = 'US', state = 'CA') INSERT INTO TABLE locationTable - PARTITION (country = 'US', state = 'AL') - SELECT * FROM another_user au - WHERE au.country = 'US' AND au.state = 'AL'; +PARTITION (country = 'US', state = 'AL') +SELECT <columns list excluding partition columns> FROM another_user </code></pre> <h4> <a id="load-data-using-dynamic-partition" class="anchor" href="#load-data-using-dynamic-partition" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Load Data Using Dynamic Partition</h4> <p>This command allows you to load data using dynamic partition. If partition spec is not specified, then the partition is considered as dynamic.</p> <p>Example:</p> -<pre><code>LOAD DATA LOCAL INPATH '${env:HOME}/staticinput.txt' - INTO TABLE locationTable +<pre><code>LOAD DATA LOCAL INPATH '${env:HOME}/staticinput.csv' +INTO TABLE locationTable INSERT INTO TABLE locationTable - SELECT * FROM another_user au - WHERE au.country = 'US' AND au.state = 'AL'; +SELECT <columns list excluding partition columns> FROM another_user </code></pre> <h4> <a id="show-partitions" class="anchor" href="#show-partitions" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Show Partitions</h4> @@ -702,17 +704,17 @@ INSERT INTO TABLE locationTable <a id="insert-overwrite" class="anchor" href="#insert-overwrite" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Insert OVERWRITE</h4> <p>This command allows you to insert or load overwrite on a spcific partition.</p> <pre><code> INSERT OVERWRITE TABLE table_name - PARTITION (column = 'partition_name') - select_statement + PARTITION (column = 'partition_name') + select_statement </code></pre> <p>Example:</p> <pre><code>INSERT OVERWRITE TABLE partitioned_user - PARTITION (country = 'US') - SELECT * FROM another_user au - WHERE au.country = 'US'; +PARTITION (country = 'US') +SELECT * FROM another_user au +WHERE au.country = 'US'; </code></pre> <h3> -<a id="carbondata-partitionhashrangelist----alpha-feature-this-partition-not-supports-update-and-delete-data" class="anchor" href="#carbondata-partitionhashrangelist----alpha-feature-this-partition-not-supports-update-and-delete-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CARBONDATA PARTITION(HASH,RANGE,LIST) -- Alpha feature, this partition not supports update and delete data.</h3> +<a id="carbondata-partitionhashrangelist----alpha-feature-this-partition-feature-does-not-support-update-and-delete-data" class="anchor" href="#carbondata-partitionhashrangelist----alpha-feature-this-partition-feature-does-not-support-update-and-delete-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CARBONDATA PARTITION(HASH,RANGE,LIST) -- Alpha feature, this partition feature does not support update and delete data.</h3> <p>The partition supports three type:(Hash,Range,List), similar to other system's partition features, CarbonData's partition feature can be used to improve query performance by filtering on the partition column.</p> <h3> <a id="create-hash-partition-table" class="anchor" href="#create-hash-partition-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create Hash Partition Table</h3> @@ -890,11 +892,11 @@ SELECT sum(price), country from sales GROUP BY country </code></pre> <p>will be transformed by Query Planner to fetch data from pre-aggregate table <strong>agg_sales</strong></p> <p>But queries of kind</p> -<pre><code>SELECT user_id, country, sex, sum(quantity), avg(price) from sales GROUP BY country, sex +<pre><code>SELECT user_id, country, sex, sum(quantity), avg(price) from sales GROUP BY user_id, country, sex SELECT sex, avg(quantity) from sales GROUP BY sex -SELECT max(price), country from sales GROUP BY country +SELECT country, max(price) from sales GROUP BY country </code></pre> <p>will fetch the data from the main table <strong>sales</strong></p> <h5> @@ -913,18 +915,12 @@ pre-aggregate tables satisfy the query condition, the plan is transformed automa pre-aggregate table to fetch the data</p> <h5> <a id="compacting-pre-aggregate-tables" class="anchor" href="#compacting-pre-aggregate-tables" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Compacting pre-aggregate tables</h5> -<p>Compaction is an optional operation for pre-aggregate table. If compaction is performed on main -table but not performed on pre-aggregate table, all queries still can benefit from pre-aggregate -table.To further improve performance on pre-aggregate table, compaction can be triggered on -pre-aggregate tables directly, it will merge the segments inside pre-aggregation table. -To do that, use ALTER TABLE COMPACT command on the pre-aggregate table just like the main table</p> -<p>NOTE:</p> -<ul> -<li>If the aggregate function used in the pre-aggregate table creation included distinct-count, -during compaction, the pre-aggregate table values are recomputed.This would a costly -operation as compared to the compaction of pre-aggregate tables containing other aggregate -functions alone</li> -</ul> +<p>Compaction command (ALTER TABLE COMPACT) need to be run separately on each pre-aggregate table. +Running Compaction command on main table will <strong>not automatically</strong> compact the pre-aggregate +tables.Compaction is an optional operation for pre-aggregate table. If compaction is performed on +main table but not performed on pre-aggregate table, all queries still can benefit from +pre-aggregate tables.To further improve performance on pre-aggregate tables, compaction can be +triggered on pre-aggregate tables directly, it will merge the segments inside pre-aggregate table.</p> <h5> <a id="updatedelete-operations-on-pre-aggregate-tables" class="anchor" href="#updatedelete-operations-on-pre-aggregate-tables" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Update/Delete Operations on pre-aggregate tables</h5> <p>This functionality is not supported.</p> @@ -1010,16 +1006,6 @@ DMPROPERTIES ( ) AS SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price), avg(price) FROM sales GROUP BY order_time, country, sex - -CREATE DATAMAP agg_minute -ON TABLE sales -USING "timeseries" -DMPROPERTIES ( -'event_time?=?order_time?, -'minute_granualrity?=?1?, -) AS -SELECT order_time, country, sex, sum(quantity), max(quantity), count(user_id), sum(price), - avg(price) FROM sales GROUP BY order_time, country, sex </code></pre> <p>For Querying data and automatically roll-up to the desired aggregation level,Carbondata supports UDF as</p> @@ -1029,9 +1015,7 @@ UDF as</p> ?hour?) </code></pre> <p>It is <strong>not necessary</strong> to create pre-aggregate tables for each granularity unless required for -query -.Carbondata -can roll-up the data and fetch it</p> +query.Carbondata can roll-up the data and fetch it.</p> <p>For Example: For main table <strong>sales</strong> , If pre-aggregate tables were created as</p> <pre><code>CREATE DATAMAP agg_day ON TABLE sales http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/faq.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/faq.html b/src/main/webapp/faq.html index 0423240..b42f8bd 100644 --- a/src/main/webapp/faq.html +++ b/src/main/webapp/faq.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/file-structure-of-carbondata.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/file-structure-of-carbondata.html b/src/main/webapp/file-structure-of-carbondata.html index 2db223a..1c8e758 100644 --- a/src/main/webapp/file-structure-of-carbondata.html +++ b/src/main/webapp/file-structure-of-carbondata.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/index.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index 40ab6b5..f2cbbbf 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -54,6 +54,9 @@ class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> @@ -319,6 +322,13 @@ </h4> <div class="linkblock"> <div class="block-row"> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a> + <span class="release-date">Feb 2018</span> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.3.0+Release"; + class="whatsnew" target="_blank">what's new</a> + </div> + <div class="block-row"> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a> <span class="release-date">Sept 2017</span> @@ -332,13 +342,6 @@ <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.1.1+Release"; class="whatsnew" target="_blank">what's new</a> </div> - <div class="block-row"> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.0/"; - target="_blank">Apache CarbonData 1.1.0</a> - <span class="release-date">May 2017</span> - <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.1.0+Release"; - class="whatsnew" target="_blank">what's new</a> - </div> </div> </div> <div class="nextR"> @@ -475,8 +478,8 @@ to do is:</p> <ol class="orderlist"> <li> - <a href="https://dist.apache.org/repos/dist/release/carbondata/1.1.1/"; - target="_blank">Download </a>the latest release. + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Download</a>the latest release. </li> <li>Follow the <a href="quick-start-guide.html">Quick Start</a>. http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/installation-guide.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/installation-guide.html b/src/main/webapp/installation-guide.html index d8c0413..7da254e 100644 --- a/src/main/webapp/installation-guide.html +++ b/src/main/webapp/installation-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/mainpage.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/mainpage.html b/src/main/webapp/mainpage.html index 52ffdb4..d782c46 100644 --- a/src/main/webapp/mainpage.html +++ b/src/main/webapp/mainpage.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> @@ -191,6 +194,7 @@ <li><a href="installation-guide.html">Installation Guide</a></li> <li><a href="configuration-parameters.html">Configuring CarbonData</a></li> + <li><a href="streaming-guide.html">Streaming Guide</a></li> <li><a href="faq.html">FAQs</a></li> <li><a href="troubleshooting.html">Troubleshooting</a></li> <li><a href="useful-tips-on-carbondata.html">Useful http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/pdf/maven-pdf-plugin.pdf ---------------------------------------------------------------------- diff --git a/src/main/webapp/pdf/maven-pdf-plugin.pdf b/src/main/webapp/pdf/maven-pdf-plugin.pdf index a59046d..72c0425 100644 Binary files a/src/main/webapp/pdf/maven-pdf-plugin.pdf and b/src/main/webapp/pdf/maven-pdf-plugin.pdf differ http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/quick-start-guide.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/quick-start-guide.html b/src/main/webapp/quick-start-guide.html index eccdabb..71ca760 100644 --- a/src/main/webapp/quick-start-guide.html +++ b/src/main/webapp/quick-start-guide.html @@ -51,6 +51,9 @@ aria-expanded="false"> Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/4a4aad2b/src/main/webapp/security.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/security.html b/src/main/webapp/security.html index 56ec55e..3822acb 100644 --- a/src/main/webapp/security.html +++ b/src/main/webapp/security.html @@ -45,6 +45,9 @@ aria-expanded="false">Download <span class="caret"></span></a> <ul class="dropdown-menu"> <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> <a href="https://dist.apache.org/repos/dist/release/carbondata/1.2.0/"; target="_blank">Apache CarbonData 1.2.0</a></li> <li>
