http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/src/main/webapp/ddl-of-carbondata.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/ddl-of-carbondata.html b/src/main/webapp/ddl-of-carbondata.html index 3e15a83..2582f4d 100644 --- a/src/main/webapp/ddl-of-carbondata.html +++ b/src/main/webapp/ddl-of-carbondata.html @@ -159,7 +159,7 @@ <div class="nav__inner"> <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> - <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a> <div class="nav__item nav__item__with__subs"> <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> @@ -179,9 +179,9 @@ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> </div> - <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a> <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> <a class="b-nav__security nav__item" href="./security.html">Security</a> @@ -194,9 +194,9 @@ <div class="b-nav__uses navindicator__item"></div> <div class="b-nav__docs navindicator__item"></div> <div class="b-nav__datamap navindicator__item"></div> - <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__api navindicator__item"></div> <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__faq navindicator__item"></div> <div class="b-nav__contri navindicator__item"></div> <div class="b-nav__security navindicator__item"></div> @@ -296,11 +296,101 @@ STORED AS carbondata <p><strong>NOTE:</strong> CarbonData also supports "STORED AS carbondata" and "USING carbondata". Find example code at <a href="https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala"; target=_blank>CarbonSessionExample</a> in the CarbonData repo.</p> <h3> <a id="usage-guidelines" class="anchor" href="#usage-guidelines" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Usage Guidelines</h3> -<p><strong>Supported properties:</strong> <a href="#dictionary-encoding-configuration">DICTIONARY_INCLUDE</a>,<a href="#inverted-index-configuration">NO_INVERTED_INDEX</a>,<a href="#sort-columns-configuration">SORT_COLUMNS</a>,<a href="#sort-scope-configuration">SORT_SCOPE</a>,<a href="#table-block-size-configuration">TABLE_BLOCKSIZE</a>,<a href="#table-compaction-configuration">MAJOR_COMPACTION_SIZE</a>,</p> -<p><a href="#table-compaction-configuration">AUTO_LOAD_MERGE</a>,<a href="#table-compaction-configuration">COMPACTION_LEVEL_THRESHOLD</a>,<a href="#table-compaction-configuration">COMPACTION_PRESERVE_SEGMENTS</a>,<a href="#table-compaction-configuration">ALLOWED_COMPACTION_DAYS</a>,</p> -<p><a href="#streaming">streaming</a>,<a href="#local-dictionary-configuration">LOCAL_DICTIONARY_ENABLE</a>,<a href="#local-dictionary-configuration">LOCAL_DICTIONARY_THRESHOLD</a>,<a href="#local-dictionary-configuration">LOCAL_DICTIONARY_INCLUDE</a>,</p> -<p><a href="#local-dictionary-configuration">LOCAL_DICTIONARY_EXCLUDE</a>,<a href="#caching-minmax-value-for-required-columns">COLUMN_META_CACHE</a>,<a href="#caching-at-block-or-blocklet-level">CACHE_LEVEL</a>,<a href="#support-flat-folder-same-as-hiveparquet">flat_folder</a>,<a href="#string-longer-than-32000-characters">LONG_STRING_COLUMNS</a>,<a href="#bucketing">BUCKETNUMBER</a>,</p> -<p><a href="#bucketing">BUCKETCOLUMNS</a></p> +<p><strong>Supported properties:</strong></p> +<table> +<thead> +<tr> +<th>Property</th> +<th>Description</th> +</tr> +</thead> +<tbody> +<tr> +<td><a href="#dictionary-encoding-configuration">DICTIONARY_INCLUDE</a></td> +<td>Columns for which dictionary needs to be generated</td> +</tr> +<tr> +<td><a href="#inverted-index-configuration">NO_INVERTED_INDEX</a></td> +<td>Columns to exclude from inverted index generation</td> +</tr> +<tr> +<td><a href="#sort-columns-configuration">SORT_COLUMNS</a></td> +<td>Columns to include in sort and its order of sort</td> +</tr> +<tr> +<td><a href="#sort-scope-configuration">SORT_SCOPE</a></td> +<td>Sort scope of the load.Options include no sort, local sort ,batch sort and global sort</td> +</tr> +<tr> +<td><a href="#table-block-size-configuration">TABLE_BLOCKSIZE</a></td> +<td>Size of blocks to write onto hdfs</td> +</tr> +<tr> +<td><a href="#table-compaction-configuration">MAJOR_COMPACTION_SIZE</a></td> +<td>Size upto which the segments can be combined into one</td> +</tr> +<tr> +<td><a href="#table-compaction-configuration">AUTO_LOAD_MERGE</a></td> +<td>Whether to auto compact the segments</td> +</tr> +<tr> +<td><a href="#table-compaction-configuration">COMPACTION_LEVEL_THRESHOLD</a></td> +<td>Number of segments to compact into one segment</td> +</tr> +<tr> +<td><a href="#table-compaction-configuration">COMPACTION_PRESERVE_SEGMENTS</a></td> +<td>Number of latest segments that needs to be excluded from compaction</td> +</tr> +<tr> +<td><a href="#table-compaction-configuration">ALLOWED_COMPACTION_DAYS</a></td> +<td>Segments generated within the configured time limit in days will be compacted, skipping others</td> +</tr> +<tr> +<td><a href="#streaming">streaming</a></td> +<td>Whether the table is a streaming table</td> +</tr> +<tr> +<td><a href="#local-dictionary-configuration">LOCAL_DICTIONARY_ENABLE</a></td> +<td>Enable local dictionary generation</td> +</tr> +<tr> +<td><a href="#local-dictionary-configuration">LOCAL_DICTIONARY_THRESHOLD</a></td> +<td>Cardinality upto which the local dictionary can be generated</td> +</tr> +<tr> +<td><a href="#local-dictionary-configuration">LOCAL_DICTIONARY_INCLUDE</a></td> +<td>Columns for which local dictionary needs to be generated.Useful when local dictionary need not be generated for all string/varchar/char columns</td> +</tr> +<tr> +<td><a href="#local-dictionary-configuration">LOCAL_DICTIONARY_EXCLUDE</a></td> +<td>Columns for which local dictionary generation should be skipped.Useful when local dictionary need not be generated for few string/varchar/char columns</td> +</tr> +<tr> +<td><a href="#caching-minmax-value-for-required-columns">COLUMN_META_CACHE</a></td> +<td>Columns whose metadata can be cached in Driver for efficient pruning and improved query performance</td> +</tr> +<tr> +<td><a href="#caching-at-block-or-blocklet-level">CACHE_LEVEL</a></td> +<td>Column metadata caching level.Whether to cache column metadata of block or blocklet</td> +</tr> +<tr> +<td><a href="#support-flat-folder-same-as-hiveparquet">flat_folder</a></td> +<td>Whether to write all the carbondata files in a single folder.Not writing segments folder during incremental load</td> +</tr> +<tr> +<td><a href="#string-longer-than-32000-characters">LONG_STRING_COLUMNS</a></td> +<td>Columns which are greater than 32K characters</td> +</tr> +<tr> +<td><a href="#bucketing">BUCKETNUMBER</a></td> +<td>Number of buckets to be created</td> +</tr> +<tr> +<td><a href="#bucketing">BUCKETCOLUMNS</a></td> +<td>Columns which are to be placed in buckets</td> +</tr> +</tbody> +</table> <p>Following are the guidelines for TBLPROPERTIES, CarbonData's additional table options can be set via carbon.properties.</p> <ul> <li> @@ -356,15 +446,15 @@ And if you care about loading resources isolation strictly, because the system u <pre><code>### Example: </code></pre> <pre><code> CREATE TABLE IF NOT EXISTS productSchema.productSalesTable ( - productNumber INT, - productName STRING, - storeCity STRING, - storeProvince STRING, - productCategory STRING, - productBatch STRING, - saleQuantity INT, - revenue INT) - STORED BY 'carbondata' + productNumber INT, + productName STRING, + storeCity STRING, + storeProvince STRING, + productCategory STRING, + productBatch STRING, + saleQuantity INT, + revenue INT) + STORED AS carbondata TBLPROPERTIES ('SORT_COLUMNS'='productName,storeCity', 'SORT_SCOPE'='NO_SORT') </code></pre> @@ -458,7 +548,7 @@ Following are 5 configurations:</p> <tr> <td>LOCAL_DICTIONARY_ENABLE</td> <td>false</td> -<td>Whether to enable local dictionary generation. <strong>NOTE:</strong> If this property is defined, it will override the value configured at system level by 'carbon.local.dictionary.enable'</td> +<td>Whether to enable local dictionary generation. <strong>NOTE:</strong> If this property is defined, it will override the value configured at system level by '<em><strong>carbon.local.dictionary.enable</strong></em>'.Local dictionary will be generated for all string/varchar/char columns unless LOCAL_DICTIONARY_INCLUDE, LOCAL_DICTIONARY_EXCLUDE is configured.</td> </tr> <tr> <td>LOCAL_DICTIONARY_THRESHOLD</td> @@ -468,12 +558,12 @@ Following are 5 configurations:</p> <tr> <td>LOCAL_DICTIONARY_INCLUDE</td> <td>string/varchar/char columns</td> -<td>Columns for which Local Dictionary has to be generated.<strong>NOTE:</strong> Those string/varchar/char columns which are added into DICTIONARY_INCLUDE option will not be considered for local dictionary generation.</td> +<td>Columns for which Local Dictionary has to be generated.<strong>NOTE:</strong> Those string/varchar/char columns which are added into DICTIONARY_INCLUDE option will not be considered for local dictionary generation.This property needs to be configured only when local dictionary needs to be generated for few columns, skipping others.This property takes effect only when <strong>LOCAL_DICTIONARY_ENABLE</strong> is true or <strong>carbon.local.dictionary.enable</strong> is true</td> </tr> <tr> <td>LOCAL_DICTIONARY_EXCLUDE</td> <td>none</td> -<td>Columns for which Local Dictionary need not be generated.</td> +<td>Columns for which Local Dictionary need not be generated.This property needs to be configured only when local dictionary needs to be skipped for few columns, generating for others.This property takes effect only when <strong>LOCAL_DICTIONARY_ENABLE</strong> is true or <strong>carbon.local.dictionary.enable</strong> is true</td> </tr> </tbody> </table> @@ -502,7 +592,7 @@ Following are 5 configurations:</p> column3 LONG ) - STORED BY 'carbondata' + STORED AS carbondata TBLPROPERTIES('LOCAL_DICTIONARY_ENABLE'='true','LOCAL_DICTIONARY_THRESHOLD'='1000', 'LOCAL_DICTIONARY_INCLUDE'='column1','LOCAL_DICTIONARY_EXCLUDE'='column2') </code></pre> @@ -613,7 +703,7 @@ You can refer to SDKwriterTestCase for example.</p> <a id="create-table-as-select" class="anchor" href="#create-table-as-select" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CREATE TABLE AS SELECT</h2> <p>This function allows user to create a Carbon table from any of the Parquet/Hive/Carbon table. This is beneficial when the user wants to create Carbon table from any other Parquet/Hive table and use the Carbon query engine to query and achieve better query results for cases where Carbon is faster than other file formats. Also this feature can be used for backing up the data.</p> <pre><code>CREATE TABLE [IF NOT EXISTS] [db_name.]table_name -STORED BY 'carbondata' +STORED AS carbondata [TBLPROPERTIES (key1=val1, key2=val2, ...)] AS select_statement; </code></pre> @@ -629,7 +719,7 @@ carbon.sql("INSERT INTO source_table SELECT 1,'bob','shenzhen',27") carbon.sql("INSERT INTO source_table SELECT 2,'david','shenzhen',31") carbon.sql("CREATE TABLE target_table - STORED BY 'carbondata' + STORED AS carbondata AS SELECT city,avg(age) FROM source_table GROUP BY city") carbon.sql("SELECT * FROM target_table").show @@ -645,21 +735,21 @@ carbon.sql("SELECT * FROM target_table").show <a id="create-external-table" class="anchor" href="#create-external-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CREATE EXTERNAL TABLE</h2> <p>This function allows user to create external table by specifying location.</p> <pre><code>CREATE EXTERNAL TABLE [IF NOT EXISTS] [db_name.]table_name -STORED BY 'carbondata' LOCATION ?$FilesPath? +STORED AS carbondata LOCATION ?$FilesPath? </code></pre> <h3> <a id="create-external-table-on-managed-table-data-location" class="anchor" href="#create-external-table-on-managed-table-data-location" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create external table on managed table data location.</h3> <p>Managed table data location provided will have both FACT and Metadata folder. This data can be generated by creating a normal carbon table and use this path as $FilesPath in the above syntax.</p> <p><strong>Example:</strong></p> -<pre><code>sql("CREATE TABLE origin(key INT, value STRING) STORED BY 'carbondata'") +<pre><code>sql("CREATE TABLE origin(key INT, value STRING) STORED AS carbondata") sql("INSERT INTO origin select 100,'spark'") sql("INSERT INTO origin select 200,'hive'") // creates a table in $storeLocation/origin sql(s""" |CREATE EXTERNAL TABLE source -|STORED BY 'carbondata' +|STORED AS carbondata |LOCATION '$storeLocation/origin' """.stripMargin) checkAnswer(sql("SELECT count(*) from source"), sql("SELECT count(*) from origin")) @@ -670,7 +760,7 @@ checkAnswer(sql("SELECT count(*) from source"), sql("SELECT count(*) from origin Our SDK module currently support writing data in this format.</p> <p><strong>Example:</strong></p> <pre><code>sql( -s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY 'carbondata' LOCATION +s"""CREATE EXTERNAL TABLE sdkOutputTable STORED AS carbondata LOCATION |'$writerPath' """.stripMargin) </code></pre> <p>Here writer path will have carbondata and index files. @@ -839,14 +929,14 @@ This can be achieved by using the alter table set command.</p> You can see the column comment of an existing table using describe formatted command.</p> <pre><code>CREATE TABLE [IF NOT EXISTS] [db_name.]table_name[(col_name data_type [COMMENT col_comment], ...)] [COMMENT table_comment] -STORED BY 'carbondata' +STORED AS carbondata [TBLPROPERTIES (property_name=property_value, ...)] </code></pre> <p>Example:</p> <pre><code>CREATE TABLE IF NOT EXISTS productSchema.productSalesTable ( productNumber Int COMMENT 'unique serial number for product') COMMENT ?This is table comment? - STORED BY 'carbondata' + STORED AS carbondata TBLPROPERTIES ('DICTIONARY_INCLUDE'='productNumber') </code></pre> <p>You can also SET and UNSET table comment using ALTER command.</p> @@ -880,7 +970,7 @@ COMMENT ?This is table comment? saleQuantity INT, revenue INT) PARTITIONED BY (productCategory STRING, productBatch STRING) -STORED BY 'carbondata' +STORED AS carbondata </code></pre> <p>NOTE: Hive partition is not supported on complex datatype columns.</p> <h4> @@ -918,7 +1008,7 @@ WHERE au.country = 'US'; <pre><code>CREATE TABLE [IF NOT EXISTS] [db_name.]table_name [(col_name data_type , ...)] PARTITIONED BY (partition_col_name data_type) -STORED BY 'carbondata' +STORED AS carbondata [TBLPROPERTIES ('PARTITION_TYPE'='HASH', 'NUM_PARTITIONS'='N' ...)] </code></pre> @@ -931,7 +1021,7 @@ STORED BY 'carbondata' col_D DECIMAL(10,2), col_F TIMESTAMP ) PARTITIONED BY (col_E LONG) -STORED BY 'carbondata' TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='9') +STORED AS carbondata TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='9') </code></pre> <h3> <a id="create-range-partition-table" class="anchor" href="#create-range-partition-table" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create Range Partition Table</h3> @@ -939,7 +1029,7 @@ STORED BY 'carbondata' TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='9 <pre><code>CREATE TABLE [IF NOT EXISTS] [db_name.]table_name [(col_name data_type , ...)] PARTITIONED BY (partition_col_name data_type) -STORED BY 'carbondata' +STORED AS carbondata [TBLPROPERTIES ('PARTITION_TYPE'='RANGE', 'RANGE_INFO'='2014-01-01, 2015-01-01, 2016-01-01, ...')] </code></pre> @@ -966,7 +1056,7 @@ STORED BY 'carbondata' <pre><code>CREATE TABLE [IF NOT EXISTS] [db_name.]table_name [(col_name data_type , ...)] PARTITIONED BY (partition_col_name data_type) -STORED BY 'carbondata' +STORED AS carbondata [TBLPROPERTIES ('PARTITION_TYPE'='LIST', 'LIST_INFO'='A, B, C, ...')] </code></pre> @@ -979,7 +1069,7 @@ STORED BY 'carbondata' col_E LONG, col_F TIMESTAMP ) PARTITIONED BY (col_A STRING) - STORED BY 'carbondata' + STORED AS carbondata TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='aaaa, bbbb, (cccc, dddd), eeee') </code></pre> @@ -1027,7 +1117,7 @@ columns to be used for bucketing and the number of buckets. For the selection of of columns is used.</p> <pre><code>CREATE TABLE [IF NOT EXISTS] [db_name.]table_name [(col_name data_type, ...)] -STORED BY 'carbondata' +STORED AS carbondata TBLPROPERTIES('BUCKETNUMBER'='noOfBuckets', 'BUCKETCOLUMNS'='columnname') </code></pre> @@ -1038,15 +1128,15 @@ TBLPROPERTIES('BUCKETNUMBER'='noOfBuckets', </ul> <p>Example:</p> <pre><code>CREATE TABLE IF NOT EXISTS productSchema.productSalesTable ( - productNumber INT, - saleQuantity INT, - productName STRING, - storeCity STRING, - storeProvince STRING, - productCategory STRING, - productBatch STRING, - revenue INT) -STORED BY 'carbondata' + productNumber INT, + saleQuantity INT, + productName STRING, + storeCity STRING, + storeProvince STRING, + productCategory STRING, + productBatch STRING, + revenue INT) +STORED AS carbondata TBLPROPERTIES ('BUCKETNUMBER'='4', 'BUCKETCOLUMNS'='productName') </code></pre> <script> @@ -1059,8 +1149,7 @@ $(function() { $('.b-nav__docs').parent().toggleClass('nav__item__with__subs--expanded'); } }); -</script> -</div> +</script></div> </div> </div> </div> @@ -1075,4 +1164,4 @@ $(function() { </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> +</html> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/src/main/webapp/dml-of-carbondata.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/dml-of-carbondata.html b/src/main/webapp/dml-of-carbondata.html index 2048c9e..ac41f7c 100644 --- a/src/main/webapp/dml-of-carbondata.html +++ b/src/main/webapp/dml-of-carbondata.html @@ -159,7 +159,7 @@ <div class="nav__inner"> <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> - <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a> <div class="nav__item nav__item__with__subs"> <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> @@ -179,9 +179,9 @@ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> </div> - <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a> <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> <a class="b-nav__security nav__item" href="./security.html">Security</a> @@ -194,9 +194,9 @@ <div class="b-nav__uses navindicator__item"></div> <div class="b-nav__docs navindicator__item"></div> <div class="b-nav__datamap navindicator__item"></div> - <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__api navindicator__item"></div> <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__faq navindicator__item"></div> <div class="b-nav__contri navindicator__item"></div> <div class="b-nav__security navindicator__item"></div> @@ -231,7 +231,101 @@ INTO TABLE [db_name.]table_name OPTIONS(property_name=property_value, ...) </code></pre> -<p><strong>Supported Properties:</strong> <a href="#delimiter">DELIMITER</a>, <a href="#quotechar">QUOTECHAR</a>, <a href="#commentchar">COMMENTCHAR</a>, <a href="#header">HEADER</a>, <a href="#fileheader">FILEHEADER</a>, <a href="#multiline">MULTILINE</a>, <a href="#escapechar">ESCAPECHAR</a>, <a href="#skip_empty_line">SKIP_EMPTY_LINE</a>, <a href="#complex_delimiter_level_1">COMPLEX_DELIMITER_LEVEL_1</a>, <a href="#complex_delimiter_level_2">COMPLEX_DELIMITER_LEVEL_2</a>, <a href="#all_dictionary_path">ALL_DICTIONARY_PATH</a>, <a href="#columndict">COLUMNDICT</a>, <a href="#dateformat">DATEFORMAT</a>,<a href="#timestampformat"> TIMESTAMPFORMAT</a>, <a href="#sort-column-bounds">SORT_COLUMN_BOUNDS</a>, <a href="#single_pass">SINGLE_PASS</a>, <a href="#bad-records-handling">BAD_RECORDS_LOGGER_ENABLE</a>, <a href="#bad-records-handling">BAD_RECORD_PATH</a>, <a href="#bad-records-handling">BAD_RECORDS_ACTION</a>, <a href="#bad-records-handling">IS_EMPTY_DATA_BAD_RECORD</a>, <a href=" #global_sort_partitions">GLOBAL_SORT_PARTITIONS</a></p> +<p><strong>Supported Properties:</strong></p> +<table> +<thead> +<tr> +<th>Property</th> +<th>Description</th> +</tr> +</thead> +<tbody> +<tr> +<td><a href="#delimiter">DELIMITER</a></td> +<td>Character used to separate the data in the input csv file</td> +</tr> +<tr> +<td><a href="#quotechar">QUOTECHAR</a></td> +<td>Character used to quote the data in the input csv file</td> +</tr> +<tr> +<td><a href="#commentchar">COMMENTCHAR</a></td> +<td>Character used to comment the rows in the input csv file.Those rows will be skipped from processing</td> +</tr> +<tr> +<td><a href="#header">HEADER</a></td> +<td>Whether the input csv files have header row</td> +</tr> +<tr> +<td><a href="#fileheader">FILEHEADER</a></td> +<td>If header is not present in the input csv, what is the column names to be used for data read from input csv</td> +</tr> +<tr> +<td><a href="#multiline">MULTILINE</a></td> +<td>Whether a row data can span across multiple lines.</td> +</tr> +<tr> +<td><a href="#escapechar">ESCAPECHAR</a></td> +<td>Escape character used to excape the data in input csv file.For eg.,\ is a standard escape character</td> +</tr> +<tr> +<td><a href="#skip_empty_line">SKIP_EMPTY_LINE</a></td> +<td>Whether empty lines in input csv file should be skipped or loaded as null row</td> +</tr> +<tr> +<td><a href="#complex_delimiter_level_1">COMPLEX_DELIMITER_LEVEL_1</a></td> +<td>Starting delimiter for complex type data in input csv file</td> +</tr> +<tr> +<td><a href="#complex_delimiter_level_2">COMPLEX_DELIMITER_LEVEL_2</a></td> +<td>Ending delimiter for complex type data in input csv file</td> +</tr> +<tr> +<td><a href="#all_dictionary_path">ALL_DICTIONARY_PATH</a></td> +<td>Path to read the dictionary data from all columns</td> +</tr> +<tr> +<td><a href="#columndict">COLUMNDICT</a></td> +<td>Path to read the dictionary data from for particular column</td> +</tr> +<tr> +<td><a href="#dateformat">DATEFORMAT</a></td> +<td>Format of date in the input csv file</td> +</tr> +<tr> +<td><a href="#timestampformat">TIMESTAMPFORMAT</a></td> +<td>Format of timestamp in the input csv file</td> +</tr> +<tr> +<td><a href="#sort-column-bounds">SORT_COLUMN_BOUNDS</a></td> +<td>How to parititon the sort columns to make the evenly distributed</td> +</tr> +<tr> +<td><a href="#single_pass">SINGLE_PASS</a></td> +<td>When to enable single pass data loading</td> +</tr> +<tr> +<td><a href="#bad-records-handling">BAD_RECORDS_LOGGER_ENABLE</a></td> +<td>Whether to enable bad records logging</td> +</tr> +<tr> +<td><a href="#bad-records-handling">BAD_RECORD_PATH</a></td> +<td>Bad records logging path.Useful when bad record logging is enabled</td> +</tr> +<tr> +<td><a href="#bad-records-handling">BAD_RECORDS_ACTION</a></td> +<td>Behavior of data loading when bad record is found</td> +</tr> +<tr> +<td><a href="#bad-records-handling">IS_EMPTY_DATA_BAD_RECORD</a></td> +<td>Whether empty data of a column to be considered as bad record or not</td> +</tr> +<tr> +<td><a href="#global_sort_partitions">GLOBAL_SORT_PARTITIONS</a></td> +<td>Number of partition to use for shuffling of data during sorting</td> +</tr> +</tbody> +</table> <p>You can use the following options to load data:</p> <ul> <li> @@ -569,8 +663,7 @@ $(function() { $('.b-nav__docs').parent().toggleClass('nav__item__with__subs--expanded'); } }); -</script> -</div> +</script></div> </div> </div> </div> @@ -585,4 +678,4 @@ $(function() { </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/src/main/webapp/documentation.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/documentation.html b/src/main/webapp/documentation.html index 634da94..982becf 100644 --- a/src/main/webapp/documentation.html +++ b/src/main/webapp/documentation.html @@ -159,7 +159,7 @@ <div class="nav__inner"> <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> - <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a> <div class="nav__item nav__item__with__subs"> <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> @@ -179,9 +179,9 @@ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> </div> - <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a> <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> <a class="b-nav__security nav__item" href="./security.html">Security</a> @@ -194,9 +194,9 @@ <div class="b-nav__uses navindicator__item"></div> <div class="b-nav__docs navindicator__item"></div> <div class="b-nav__datamap navindicator__item"></div> - <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__api navindicator__item"></div> <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__faq navindicator__item"></div> <div class="b-nav__contri navindicator__item"></div> <div class="b-nav__security navindicator__item"></div> @@ -215,13 +215,13 @@ <p>Apache CarbonData is a new big data file format for faster interactive query using advanced columnar storage, index, compression and encoding techniques to improve computing efficiency, which helps in speeding up queries by an order of magnitude faster over PetaBytes of data.</p> <h2> <a id="getting-started" class="anchor" href="#getting-started" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Getting Started</h2> -<p><strong>File Format Concepts:</strong> Start with the basics of understanding the <a href="./file-structure-of-carbondata.html#carbondata-file-structure">CarbonData file format</a> and its storage structure.This will help to understand other parts of the documentation, incuding deployment, programming and usage guides.</p> +<p><strong>File Format Concepts:</strong> Start with the basics of understanding the <a href="./file-structure-of-carbondata.html#carbondata-file-format">CarbonData file format</a> and its <a href="./file-structure-of-carbondata.html">storage structure</a>.This will help to understand other parts of the documentation, including deployment, programming and usage guides.</p> <p><strong>Quick Start:</strong> <a href="./quick-start-guide.html#installing-and-configuring-carbondata-to-run-locally-with-spark-shell">Run an example program</a> on your local machine or <a href="https://github.com/apache/carbondata/tree/master/examples/spark2/src/main/scala/org/apache/carbondata/examples"; target=_blank>study some examples</a>.</p> <p><strong>CarbonData SQL Language Reference:</strong> CarbonData extends the Spark SQL language and adds several <a href="./ddl-of-carbondata.html">DDL</a> and <a href="./dml-of-carbondata.html">DML</a> statements to support operations on it.Refer to the <a href="./language-manual.html">Reference Manual</a> to understand the supported features and functions.</p> <p><strong>Programming Guides:</strong> You can read our guides about <a href="./sdk-guide.html">APIs supported</a> to learn how to integrate CarbonData with your applications.</p> <h2> -<a id="deployment" class="anchor" href="#deployment" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Deployment</h2> -<p>CarbonData can be integrated with popular Execution engines like <a href="./quick-start-guide.html#spark">Spark</a> and <a href="./quick-start-guide.html#presto">Presto</a>.Refer to the <a href="./quick-start-guide.html##deployment-modes">Installation and Configuration</a> section to understand all modes of Integrating CarbonData.</p> +<a id="integration" class="anchor" href="#integration" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Integration</h2> +<p>CarbonData can be integrated with popular Execution engines like <a href="./quick-start-guide.html#spark">Spark</a> and <a href="./quick-start-guide.html#presto">Presto</a>.Refer to the <a href="./quick-start-guide.html#integration">Installation and Configuration</a> section to understand all modes of Integrating CarbonData.</p> <h2> <a id="contributing-to-carbondata" class="anchor" href="#contributing-to-carbondata" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Contributing to CarbonData</h2> <p>The Apache CarbonData community welcomes all kinds of contributions from anyone with a passion for @@ -234,10 +234,6 @@ faster data format.Contributing to CarbonData doesn?t just mean writing code. He <p><strong>Blogs:</strong> Blogs by external users can be found <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=67635497"; rel="nofollow">here</a>.</p> <p><strong>Performance reports:</strong> TPC-H performance reports can be found <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Performance+-+TPCH+Report+of+CarbonData+%281.2+version%29+and+Parquet+on+Spark+Execution+Engine"; rel="nofollow">here</a>.</p> <p><strong>Trainings:</strong> Training records on design and code flows can be found <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/CarbonData+Training+Materials"; rel="nofollow">here</a>.</p> -<script> -// Show selected style on nav item -$(function() { $('.b-nav__intro').addClass('selected'); }); -</script> </div> </div> </div> @@ -253,4 +249,4 @@ $(function() { $('.b-nav__intro').addClass('selected'); }); </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/src/main/webapp/errorpage.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/errorpage.html b/src/main/webapp/errorpage.html index 35cc01a..090dce5 100644 --- a/src/main/webapp/errorpage.html +++ b/src/main/webapp/errorpage.html @@ -89,4 +89,4 @@ </body> -</html> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/src/main/webapp/faq.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/faq.html b/src/main/webapp/faq.html index 52112b7..c37284f 100644 --- a/src/main/webapp/faq.html +++ b/src/main/webapp/faq.html @@ -159,7 +159,7 @@ <div class="nav__inner"> <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> - <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a> <div class="nav__item nav__item__with__subs"> <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> @@ -179,9 +179,9 @@ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> </div> - <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a> <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> <a class="b-nav__security nav__item" href="./security.html">Security</a> @@ -194,9 +194,9 @@ <div class="b-nav__uses navindicator__item"></div> <div class="b-nav__docs navindicator__item"></div> <div class="b-nav__datamap navindicator__item"></div> - <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__api navindicator__item"></div> <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__faq navindicator__item"></div> <div class="b-nav__contri navindicator__item"></div> <div class="b-nav__security navindicator__item"></div> @@ -222,19 +222,19 @@ <li><a href="#how-to-resolve-abstract-method-error">How to resolve Abstract Method Error?</a></li> <li><a href="#how-carbon-will-behave-when-execute-insert-operation-in-abnormal-scenarios">How Carbon will behave when execute insert operation in abnormal scenarios?</a></li> <li><a href="#why-aggregate-query-is-not-fetching-data-from-aggregate-table">Why aggregate query is not fetching data from aggregate table?</a></li> -<li><a href="#Why-all-executors-are-showing-success-in-Spark-UI-even-after-Dataload-command-failed-at-driver-side">Why all executors are showing success in Spark UI even after Dataload command failed at Driver side?</a></li> -<li><a href="#Why-different-time-zone-result-for-select-query-output-when-query-SDK-writer-output">Why different time zone result for select query output when query SDK writer output?</a></li> +<li><a href="#why-all-executors-are-showing-success-in-spark-ui-even-after-dataload-command-failed-at-driver-side">Why all executors are showing success in Spark UI even after Dataload command failed at Driver side?</a></li> +<li><a href="#why-different-time-zone-result-for-select-query-output-when-query-sdk-writer-output">Why different time zone result for select query output when query SDK writer output?</a></li> </ul> <h1> <a id="troubleshooting" class="anchor" href="#troubleshooting" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>TroubleShooting</h1> <ul> -<li><a href="#Getting-tablestatus.lock-issues-When-loading-data">Getting tablestatus.lock issues When loading data</a></li> +<li><a href="#getting-tablestatuslock-issues-when-loading-data">Getting tablestatus.lock issues When loading data</a></li> <li><a href="#failed-to-load-thrift-libraries">Failed to load thrift libraries</a></li> <li><a href="#failed-to-launch-the-spark-shell">Failed to launch the Spark Shell</a></li> <li><a href="#failed-to-execute-load-query-on-cluster">Failed to execute load query on cluster</a></li> <li><a href="#failed-to-execute-insert-query-on-cluster">Failed to execute insert query on cluster</a></li> <li><a href="#failed-to-connect-to-hiveuser-with-thrift">Failed to connect to hiveuser with thrift</a></li> -<li><a href="#failed-to-read-the-metastore-db-during-table">Failed to read the metastore db during table</a></li> +<li><a href="#failed-to-read-the-metastore-db-during-table-creation">Failed to read the metastore db during table creation</a></li> <li><a href="#failed-to-load-data-on-the-cluster">Failed to load data on the cluster</a></li> <li><a href="#failed-to-insert-data-on-the-cluster">Failed to insert data on the cluster</a></li> <li><a href="#failed-to-execute-concurrent-operations-on-table-by-multiple-workers">Failed to execute Concurrent Operations(Load,Insert,Update) on table by multiple workers</a></li> @@ -292,7 +292,7 @@ The property carbon.lock.type configuration specifies the type of lock to be acq <p>In order to build CarbonData project it is necessary to specify the spark profile. The spark profile sets the Spark Version. You need to specify the <code>spark version</code> while using Maven to build project.</p> <h2> <a id="how-carbon-will-behave-when-execute-insert-operation-in-abnormal-scenarios" class="anchor" href="#how-carbon-will-behave-when-execute-insert-operation-in-abnormal-scenarios" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>How Carbon will behave when execute insert operation in abnormal scenarios?</h2> -<p>Carbon support insert operation, you can refer to the syntax mentioned in <a href="dml-operation-on-carbondata.html">DML Operations on CarbonData</a>. +<p>Carbon support insert operation, you can refer to the syntax mentioned in <a href="./dml-of-carbondata.html">DML Operations on CarbonData</a>. First, create a source table in spark-sql and load data into this created table.</p> <pre><code>CREATE TABLE source_table( id String, @@ -312,7 +312,7 @@ id name city id String, city String, name String) -STORED BY 'carbondata'; +STORED AS carbondata; </code></pre> <pre><code>INSERT INTO TABLE carbon_table SELECT * FROM source_table; </code></pre> @@ -341,7 +341,7 @@ id city name When SubQuery predicate is present in the query.</li> </ul> <p>Example:</p> -<pre><code>create table gdp21(cntry smallint, gdp double, y_year date) stored by 'carbondata'; +<pre><code>create table gdp21(cntry smallint, gdp double, y_year date) stored as carbondata; create datamap ag1 on table gdp21 using 'preaggregate' as select cntry, sum(gdp) from gdp21 group by cntry; select ctry from pop1 where ctry in (select cntry from gdp21 group by cntry); </code></pre> @@ -351,7 +351,7 @@ select ctry from pop1 where ctry in (select cntry from gdp21 group by cntry); When aggregate function along with 'in' filter.</li> </ul> <p>Example:</p> -<pre><code>create table gdp21(cntry smallint, gdp double, y_year date) stored by 'carbondata'; +<pre><code>create table gdp21(cntry smallint, gdp double, y_year date) stored as carbondata; create datamap ag1 on table gdp21 using 'preaggregate' as select cntry, sum(gdp) from gdp21 group by cntry; select cntry, sum(gdp) from gdp21 where cntry in (select ctry from pop1) group by cntry; </code></pre> @@ -361,7 +361,7 @@ select cntry, sum(gdp) from gdp21 where cntry in (select ctry from pop1) group b When aggregate function having 'join' with equal filter.</li> </ul> <p>Example:</p> -<pre><code>create table gdp21(cntry smallint, gdp double, y_year date) stored by 'carbondata'; +<pre><code>create table gdp21(cntry smallint, gdp double, y_year date) stored as carbondata; create datamap ag1 on table gdp21 using 'preaggregate' as select cntry, sum(gdp) from gdp21 group by cntry; select cntry,sum(gdp) from gdp21,pop1 where cntry=ctry group by cntry; </code></pre> @@ -566,8 +566,7 @@ For example, you can use scp to copy this file to all the nodes.</p> <script> // Show selected style on nav item $(function() { $('.b-nav__faq').addClass('selected'); }); -</script> -</div> +</script></div> </div> </div> </div> @@ -582,4 +581,4 @@ $(function() { $('.b-nav__faq').addClass('selected'); }); </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/src/main/webapp/file-structure-of-carbondata.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/file-structure-of-carbondata.html b/src/main/webapp/file-structure-of-carbondata.html index b96d622..c14ea6d 100644 --- a/src/main/webapp/file-structure-of-carbondata.html +++ b/src/main/webapp/file-structure-of-carbondata.html @@ -159,7 +159,7 @@ <div class="nav__inner"> <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> - <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a> <div class="nav__item nav__item__with__subs"> <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> @@ -179,9 +179,9 @@ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> </div> - <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a> <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> <a class="b-nav__security nav__item" href="./security.html">Security</a> @@ -194,9 +194,9 @@ <div class="b-nav__uses navindicator__item"></div> <div class="b-nav__docs navindicator__item"></div> <div class="b-nav__datamap navindicator__item"></div> - <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__api navindicator__item"></div> <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__faq navindicator__item"></div> <div class="b-nav__contri navindicator__item"></div> <div class="b-nav__security navindicator__item"></div> @@ -211,30 +211,138 @@ <div class="col-sm-12 col-md-12"> <div> <h1> -<a id="carbondata-file-structure" class="anchor" href="#carbondata-file-structure" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData File Structure</h1> +<a id="carbondata-table-structure" class="anchor" href="#carbondata-table-structure" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData table structure</h1> <p>CarbonData files contain groups of data called blocklets, along with all required information like schema, offsets and indices etc, in a file header and footer, co-located in HDFS.</p> <p>The file footer can be read once to build the indices in memory, which can be utilized for optimizing the scans and processing for all subsequent queries.</p> -<h3> -<a id="understanding-carbondata-file-structure" class="anchor" href="#understanding-carbondata-file-structure" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Understanding CarbonData File Structure</h3> +<p>This document describes the what a CarbonData table looks like in a HDFS directory, files written and content of each file.</p> <ul> -<li>Block : It would be as same as HDFS block, CarbonData creates one file for each data block, user can specify TABLE_BLOCKSIZE during creation table. Each file contains File Header, Blocklets and File Footer.</li> -</ul> -<p><a href="../docs/images/carbon_data_file_structure_new.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/carbon_data_file_structure_new.png?raw=true"; alt="CarbonData File Structure" style="max-width:100%;"></a></p> +<li> +<p><a href="#file-directory-structure">File Directory Structure</a></p> +</li> +<li> +<p><a href="#file-content-details">File Content details</a></p> <ul> -<li>File Header : It contains CarbonData file version number, list of column schema and schema updation timestamp.</li> -<li>File Footer : it contains Number of rows, segmentinfo ,all blocklets? info and index, you can find the detail from the below diagram.</li> -<li>Blocklet : Rows are grouped to form a blocklet, the size of the blocklet is configurable and default size is 64MB, Blocklet contains Column Page groups for each column.</li> -<li>Column Page Group : Data of one column and it is further divided into pages, it is guaranteed to be contiguous in file.</li> -<li>Page : It has the data of one column and the number of row is fixed to 32000 size.</li> -</ul> -<p><a href="../docs/images/carbon_data_format_new.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/carbon_data_format_new.png?raw=true"; alt="CarbonData File Format" style="max-width:100%;"></a></p> -<h3> -<a id="each-page-contains-three-types-of-data" class="anchor" href="#each-page-contains-three-types-of-data" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Each page contains three types of data</h3> +<li><a href="#schema-file-format">Schema file format</a></li> +<li> +<a href="#carbondata-file-format">CarbonData file format</a> <ul> -<li>Data Page: Contains the encoded data of a column of columns.</li> -<li>Row ID Page (optional): Contains the row ID mappings used when the data page is stored as an inverted index.</li> -<li>RLE Page (optional): Contains additional metadata used when the data page is RLE coded.</li> +<li> +<a href="#blocklet-format">Blocklet format</a> +<ul> +<li><a href="#v1">V1</a></li> +<li><a href="#v2">V2</a></li> +<li><a href="#v3">V3</a></li> +</ul> +</li> +<li><a href="#footer-format">Footer format</a></li> </ul> +</li> +<li><a href="#carbonindex-file-format">carbonindex file format</a></li> +<li><a href="#dictionary-file-format">Dictionary file format</a></li> +<li><a href="#tablestatus-file-format">tablestatus file format</a></li> +</ul> +</li> +</ul> +<h2> +<a id="file-directory-structure" class="anchor" href="#file-directory-structure" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>File Directory Structure</h2> +<p>The CarbonData files are stored in the location specified by the <em><strong>carbon.storelocation</strong></em> configuration (configured in carbon.properties; if not configured, the default is ../carbon.store).</p> +<p>The file directory structure is as below:</p> +<p><a href="../docs/images/2-1_1.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-1_1.png?raw=true"; alt="File Directory Structure" style="max-width:100%;"></a></p> +<ol> +<li>ModifiedTime.htmlt records the timestamp of the metadata with the modification time attribute of the file. When the drop table and create table are used, the modification time of the file is updated.This is common to all databases and hence is kept in parallel to databases</li> +<li>The <strong>default</strong> is the database name and contains the user tables.default is used when user doesn't specify any database name;else user configured database name will be the directory name. user_table is the table name.</li> +<li>Metadata directory stores schema files, tablestatus and dictionary files (including .dict, .dictmeta and .sortindex). There are three types of metadata data information files.</li> +<li>data and index files are stored under directory named <strong>Fact</strong>. The Fact directory has a Part0 partition directory, where 0 is the partition number.</li> +<li>There is a Segment_0 directory under the Part0 directory, where 0 is the segment number.</li> +<li>There are two types of files, carbondata and carbonindex, in the Segment_0 directory.</li> +</ol> +<h2> +<a id="file-content-details" class="anchor" href="#file-content-details" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>File Content details</h2> +<p>When the table is created, the user_table directory is generated, and a schema file is generated in the Metadata directory for recording the table structure.</p> +<p>When loading data in batches, each batch loading generates a new segment directory. The scheduling tries to control a task processing data loading task on each node. Each task will generate multiple carbondata files and one carbonindex file.</p> +<p>During global dictionary generation, if the two-pass scheme is used, before the data is loaded, the corresponding dict, dictmeta and sortindex files are generated for each dictionary-encoded column, and partial dictionary files can be provided by the pre-define dictionary method to reduce the need. A dictionary-encoded column is generated by scanning the full amount of data; a dictionary file of all dictionary code columns can also be provided by the all dictionary method to avoid scanning data. If the single-pass scheme is adopted, the global dictionary code is generated in real time during data loading, and after the data is loaded, the dictionary is solidified into a dictionary file.</p> +<p>The following sections use the Java object generated by the thrift file describing the carbondata file format to explain the contents of each file one by one (you can also directly read the format defined in the <a href="https://github.com/apache/carbondata/tree/master/format/src/main/thrift"; target=_blank>thrift file</a>)</p> +<h3> +<a id="schema-file-format" class="anchor" href="#schema-file-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Schema file format</h3> +<p>The contents of the schema file is as shown below</p> +<p><a href="../docs/images/2-2_1.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-2_1.png?raw=true"; alt="Schema file format" style="max-width:100%;"></a></p> +<ol> +<li>TableSchema class +The TableSchema class does not store the table name, it is infered from the directory name(user_table). +tableProperties is used to record table-related properties, such as: table_blocksize.</li> +<li>ColumnSchema class +Encoders are used to record the encoding used in column storage. +columnProperties is used to record column related properties.</li> +<li>BucketingInfo class +When creating a bucket table, you can specify the number of buckets in the table and the column to splitbuckets.</li> +<li>DataType class +Describes the data types supported by CarbonData.</li> +<li>Encoding class +Several encodings that may be used in CarbonData files.</li> +</ol> +<h3> +<a id="carbondata-file-format" class="anchor" href="#carbondata-file-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData file format</h3> +<h4> +<a id="file-header" class="anchor" href="#file-header" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>File Header</h4> +<p>It contains CarbonData file version number, list of column schema and schema updation timestamp.</p> +<p><a href="../docs/images/carbon_data_file_structure_new.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/carbon_data_file_structure_new.png?raw=true"; alt="File Header" style="max-width:100%;"></a></p> +<p>The carbondata file consists of multiple blocklets and footer parts. The blocklet is the dataset inside the carbondata file (the latest V3 format, the default configuration is 64MB), each blocklet contains a ColumnChunk for each column, and a ColumnChunk may contain one or more Column Pages.</p> +<p>The carbondata file currently supports V1, V2 and V3 versions. The main difference is the change of the blocklet part, which is introduced one by one.</p> +<h4> +<a id="blocklet-format" class="anchor" href="#blocklet-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Blocklet format</h4> +<h5> +<a id="v1" class="anchor" href="#v1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>V1</h5> +<p>Blocket consists of all column data pages, RLE pages, and rowID pages. Since the pages in the blocklet are grouped according to the page type, the three pieces of data of each column are distributed and stored in the blocklet, and the offset and length information of all the pages need to be recorded in the footer part.</p> +<p><a href="../docs/images/2-3_1.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-3_1.png?raw=true"; alt="V1" style="max-width:100%;"></a></p> +<h5> +<a id="v2" class="anchor" href="#v2" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>V2</h5> +<p>The blocklet consists of ColumnChunk for all columns. The ColumnChunk for a column consists of a ColumnPage, which includes the data chunk header, data page, RLE page, and rowID page. Since ColumnChunk aggregates the three types of Page data of the column together, it can read the column data using fewer readers. Since the header part records the length information of all the pages, the footer part only needs to record the offset and length of the ColumnChunk, and also reduces the amount of footer data.</p> +<p><a href="../docs/images/2-3_2.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-3_2.png?raw=true"; alt="V2" style="max-width:100%;"></a></p> +<h5> +<a id="v3" class="anchor" href="#v3" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>V3</h5> +<p>The blocklet is also composed of ColumnChunks of all columns. What is changed is that a ColumnChunk consists of one or more Column Pages, and Column Page adds a new BlockletMinMaxIndex.</p> +<p>Compared with V2: The blocklet data volume of V2 format defaults to 120,000 lines, and the blocklet data volume of V3 format defaults to 64MB. For the same size data file, the information of the footer part index metadata may be further reduced; meanwhile, the V3 format adds a new page. Level data filtering, and the amount of data per page is only 32,000 lines by default, which is much less than the 120,000 lines of V2 format. The accuracy of data filtering hits further, and more data can be filtered out before decompressing data.</p> +<p><a href="../docs/images/2-3_3.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-3_3.png?raw=true"; alt="V3" style="max-width:100%;"></a></p> +<h4> +<a id="footer-format" class="anchor" href="#footer-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Footer format</h4> +<p>Footer records each carbondata +All blocklet data distribution information and statistical related metadata information (minmax, startkey/endkey) inside the file.</p> +<p><a href="../docs/images/2-3_4.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-3_4.png?raw=true"; alt="Footer format" style="max-width:100%;"></a></p> +<ol> +<li>BlockletInfo3 is used to record the offset and length of all ColumnChunk3.</li> +<li>SegmentInfo is used to record the number of columns and the cardinality of each column.</li> +<li>BlockletIndex includes BlockletMinMaxIndex and BlockletBTreeIndex.</li> +</ol> +<p>BlockletBTreeIndex is used to record the startkey/endkey of all blocklets in the block. When querying, the startkey/endkey of the query is generated by filtering conditions combined with.htmlkey. With BlocketBtreeIndex, the range of blocklets satisfying the conditions in each block can be delineated.</p> +<p>BlockletMinMaxIndex is used to record the min/max value of all columns in the blocklet. By using the min/max check on the filter condition, you can skip the block/blocklet that does not satisfy the condition.</p> +<h3> +<a id="carbonindex-file-format" class="anchor" href="#carbonindex-file-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>carbonindex file format</h3> +<p>Extract the BlockletIndex part of the footer part to generate the carbonindex file. Load data in batches, schedule as much as possible to control a node to start a task, each task generates multiple carbondata files and a carbonindex file. The carbonindex file records the index information of all the blocklets in all the carbondata files generated by the task.</p> +<p>As shown in the figure, the index information corresponding to a block is recorded by a BlockIndex object, including carbondata filename, footer offset and BlockletIndex. The BlockIndex data volume is less than the footer. The file is directly used to build the index on the driver side when querying, without having to skip the footer part of the data volume of multiple data files.</p> +<p><a href="../docs/images/2-4_1.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-4_1.png?raw=true"; alt="carbonindex file format" style="max-width:100%;"></a></p> +<h3> +<a id="dictionary-file-format" class="anchor" href="#dictionary-file-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Dictionary file format</h3> +<p>For each dictionary encoded column, a dictionary file is used to store the dictionary metadata for that column.</p> +<ol> +<li>dict file records the distinct value list of a column</li> +</ol> +<p>For the first time dataloading, the file is generated using a distinct value list of a column. The value in the file is unordered; the subsequent append is used. In the second step of dataloading (Data Convert Step), the dictionary code column will replace the true value of the data with the dictionary key.</p> +<p><a href="../docs/images/2-5_1.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-5_1.png?raw=true"; alt="Dictionary file format" style="max-width:100%;"></a></p> +<ol start="2"> +<li>dictmeta records the metadata description of the new distinct value of each dataloading</li> +</ol> +<p>The dictionary cache uses this information to incrementally flush the cache.</p> +<p><a href="../docs/images/2-5_2.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-5_2.png?raw=true"; alt="Dictionary Chunk" style="max-width:100%;"></a></p> +<ol start="3"> +<li>sortindex records the result set of the key code of the dictionary code sorted by value.</li> +</ol> +<p>In dataLoading, if there is a new dictionary value, the sortindex file will be regenerated using all the dictionary codes.</p> +<p>Filtering queries based on dictionary code columns need to convert the value filter filter to the key filter condition. Using the sortindex file, you can quickly construct an ordered value sequence to quickly find the key value corresponding to the value, thus speeding up the conversion process.</p> +<p><a href="../docs/images/2-5_3.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-5_3.png?raw=true"; alt="sortindex file format" style="max-width:100%;"></a></p> +<h3> +<a id="tablestatus-file-format" class="anchor" href="#tablestatus-file-format" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>tablestatus file format</h3> +<p>Tablestatus records the segment-related information (in gson format) for each load and merge, including load time, load status, segment name, whether it was deleted, and the segment name incorporated. Regenerate the tablestatusfile after each load or merge.</p> +<p><a href="../docs/images/2-6_1.png?raw=true" target="_blank" rel="noopener noreferrer"><img src="https://github.com/apache/carbondata/blob/master/docs/images/2-6_1.png?raw=true"; alt="tablestatus file format" style="max-width:100%;"></a></p> <script> $(function() { // Show selected style on nav item @@ -245,8 +353,7 @@ $(function() { $('.b-nav__docs').parent().toggleClass('nav__item__with__subs--expanded'); } }); -</script> -</div> +</script></div> </div> </div> </div> @@ -261,4 +368,4 @@ $(function() { </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/src/main/webapp/how-to-contribute-to-apache-carbondata.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/how-to-contribute-to-apache-carbondata.html b/src/main/webapp/how-to-contribute-to-apache-carbondata.html index 946013f..122b763 100644 --- a/src/main/webapp/how-to-contribute-to-apache-carbondata.html +++ b/src/main/webapp/how-to-contribute-to-apache-carbondata.html @@ -159,7 +159,7 @@ <div class="nav__inner"> <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> - <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + <a class="b-nav__uses nav__item" href="./usecases.html">use cases</a> <div class="nav__item nav__item__with__subs"> <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> @@ -179,9 +179,9 @@ <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> </div> - <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a> <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> <a class="b-nav__security nav__item" href="./security.html">Security</a> @@ -194,9 +194,9 @@ <div class="b-nav__uses navindicator__item"></div> <div class="b-nav__docs navindicator__item"></div> <div class="b-nav__datamap navindicator__item"></div> - <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__api navindicator__item"></div> <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> <div class="b-nav__faq navindicator__item"></div> <div class="b-nav__contri navindicator__item"></div> <div class="b-nav__security navindicator__item"></div> @@ -352,8 +352,7 @@ $ git push <GitHub_user> --delete <my-branch> <script> // Show selected style on nav item $(function() { $('.b-nav__contri').addClass('selected'); }); -</script> -</div> +</script></div> </div> </div> </div> @@ -368,4 +367,4 @@ $(function() { $('.b-nav__contri').addClass('selected'); }); </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/a51dc596/src/main/webapp/index.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index e9e6e3b..f059d16 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -477,7 +477,7 @@ </ol> <p class="title-info"> For detailed reference on CarbonData, read the <a - href="mainpage.html">User Guide</a>. + href="documentation.html">User Guide</a>. </p> </div>
![https://github.com/apache/carbondata/blob/master/docs/images/carbon_data_file_structure_new.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/carbon_data_file_structure_new.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/carbon_data_format_new.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/carbon_data_format_new.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-1_1.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-1_1.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-2_1.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-2_1.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-3_1.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-3_1.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-3_2.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-3_2.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-3_3.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-3_3.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-3_4.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-3_4.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-4_1.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-4_1.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-5_1.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-5_1.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-5_2.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-5_2.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-5_3.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-5_3.png?raw=true"){kind=link}
![https://github.com/apache/carbondata/blob/master/docs/images/2-6_1.png?raw=true"](https://github.com/apache/carbondata/blob/master/docs/images/2-6_1.png?raw=true"){kind=link}