Added: tajo/site/docs/devel/table_management/json.html URL: http://svn.apache.org/viewvc/tajo/site/docs/devel/table_management/json.html?rev=1706955&view=auto ============================================================================== --- tajo/site/docs/devel/table_management/json.html (added) +++ tajo/site/docs/devel/table_management/json.html Tue Oct 6 06:45:04 2015 @@ -0,0 +1,363 @@ + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>JSON — Apache Tajo 0.11.0 documentation</title> + + + + + + + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + + + + <link rel="top" title="Apache Tajo 0.11.0 documentation" href="../index.html"/> + <link rel="up" title="File Formats" href="file_formats.html"/> + <link rel="next" title="RCFile" href="rcfile.html"/> + <link rel="prev" title="TEXT" href="text.html"/> + + + <script src="https://cdnjs.cloudflare.com/ajax/libs/modernizr/2.6.2/modernizr.min.js";></script> + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-nav-search"> + <a href="../index.html" class="fa fa-home"> Apache Tajo</a> + <div role="search"> + <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + <ul class="current"> +<li class="toctree-l1"><a class="reference internal" href="../introduction.html">Introduction</a></li> +<li class="toctree-l1"><a class="reference internal" href="../getting_started.html">Getting Started</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#prerequisites">Prerequisites</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#dowload-and-unpack-the-source-code">Dowload and unpack the source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#build-source-code">Build source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#setting-up-a-local-tajo-cluster">Setting up a local Tajo cluster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#first-query-execution">First query execution</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../configuration/preliminary.html">Preliminary</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/cluster_setup.html">Cluster Setup</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo_master_configuration.html">Tajo Master Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/worker_configuration.html">Worker Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog_configuration.html">Catalog Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/ha_configuration.html">High Availability for TajoMaster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/service_config_defaults.html">Cluster Service Configuration Defaults</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo-site-xml.html">The tajo-site.xml File</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog-site-xml.html">The catalog-site.xml File</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tsql.html">Tajo Shell (TSQL)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../tsql/meta_command.html">Meta Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/dfs_command.html">Executing HDFS commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/variables.html">Session Variables</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/admin_command.html">Administration Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/intro.html">Introducing to TSQL</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/single_command.html">Executing a single command</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/execute_file.html">Executing Queries from Files</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/background_command.html">Executing as background process</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../sql_language.html">SQL Language</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/data_model.html">Data Model</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/ddl.html">Data Definition Language</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/insert.html">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/alter_table.html">ALTER TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/queries.html">Queries</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/joins.html">Joins</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/sql_expression.html">SQL Expressions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/predicates.html">Predicates</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../time_zone.html">Time Zone</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#server-cluster-time-zone">Server Cluster Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#table-time-zone">Table Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#client-time-zone">Client Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#time-zone-id">Time Zone ID</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#examples-of-time-zone">Examples of Time Zone</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../functions.html">Functions</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../functions.html#built-in-scalar-functions">Built-in Scalar Functions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions.html#built-in-aggregation-functions">Built-in Aggregation Functions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions.html#built-in-window-functions">Built-in Window Functions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions.html#user-defined-functions">User-defined Functions</a></li> +</ul> +</li> +<li class="toctree-l1 current"><a class="reference internal" href="../table_management.html">Table Management</a><ul class="current"> +<li class="toctree-l2"><a class="reference internal" href="table_overview.html">Overview of Tajo Tables</a></li> +<li class="toctree-l2"><a class="reference internal" href="tablespaces.html">Tablespaces</a></li> +<li class="toctree-l2 current"><a class="reference internal" href="file_formats.html">File Formats</a></li> +<li class="toctree-l2"><a class="reference internal" href="compression.html">Compression</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../table_partitioning.html">Table Partitioning</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/intro_to_partitioning.html">Introduction to Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/column_partitioning.html">Column Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/range_partitioning.html">Range Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/hash_partitioning.html">Hash Partitioning</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../storage_plugins.html">Storage Plugin</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../storage_plugins/overview.html">Storage Plugin Overview</a></li> +<li class="toctree-l2"><a class="reference internal" href="../storage_plugins/postgresql.html">PostgreSQL Storage Handler</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../index_overview.html">Index (Experimental Feature)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../index/types.html">Index Types</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/how_to_use.html">How to use index?</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/future_work.html">Future Works</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../backup_and_restore.html">Backup and Restore</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../backup_and_restore/catalog.html">Backup and Restore Catalog</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../hive_integration.html">Hive Integration</a></li> +<li class="toctree-l1"><a class="reference internal" href="../hbase_integration.html">HBase Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#create-table">CREATE TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#drop-table">DROP TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#insert-overwrite-into">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#usage">Usage</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../swift_integration.html">OpenStack Swift Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#swift-configuration">Swift configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#hadoop-configurations">Hadoop configurations</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#tajo-configuration">Tajo configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#querying-on-swift">Querying on Swift</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../jdbc_driver.html">Tajo JDBC Driver</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#how-to-get-jdbc-driver">How to get JDBC driver</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#setting-the-classpath">Setting the CLASSPATH</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#connecting-to-the-tajo-cluster-instance">Connecting to the Tajo cluster instance</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#connection-parameters">Connection Parameters</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#an-example-jdbc-client">An Example JDBC Client</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tajo_client_api.html">Tajo Client API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> +</ul> + + + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Tajo</a> + </nav> + + + + <div class="wy-nav-content"> + <div class="rst-content"> + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="../table_management.html">Table Management</a> »</li> + + <li><a href="file_formats.html">File Formats</a> »</li> + + <li>JSON</li> + <li class="wy-breadcrumbs-aside"> + + <a href="../_sources/table_management/json.txt" rel="nofollow"> View page source</a> + + </li> + </ul> + <hr/> +</div> + <div role="main"> + + <div class="section" id="json"> +<h1>JSON<a class="headerlink" href="#json" title="Permalink to this headline">¶</a></h1> +<p>JSON(JavaScript Object Notation) is an open standard format for data (de)serialization. Since it is simple and human-readable, it is popularly used in many fields. +Tajo supports JSON as its data format. In this section, you will get an overview of how to create JSON tables and query on them.</p> +<div class="section" id="how-to-create-a-json-table"> +<h2>How to Create a JSON Table ?<a class="headerlink" href="#how-to-create-a-json-table" title="Permalink to this headline">¶</a></h2> +<p>You can create a JSON table using the <code class="docutils literal"><span class="pre">CREATE</span> <span class="pre">TABLE</span></code> statement. (For more information, please refer to <a class="reference internal" href="../sql_language/ddl.html"><em>Data Definition Language</em></a>.) +For example, please consider an example data as follows:</p> +<div class="highlight-bash"><div class="highlight"><pre><span class="nv">$ </span>hdfs dfs -cat /table1/table.json +<span class="o">{</span> <span class="s2">"title"</span> : <span class="s2">"Hand of the King"</span>, <span class="s2">"name"</span> : <span class="o">{</span> <span class="s2">"first_name"</span>: <span class="s2">"Eddard"</span>, <span class="s2">"last_name"</span>: <span class="s2">"Stark"</span><span class="o">}}</span> +<span class="o">{</span> <span class="s2">"title"</span> : <span class="s2">"Assassin"</span>, <span class="s2">"name"</span> : <span class="o">{</span> <span class="s2">"first_name"</span>: <span class="s2">"Arya"</span>, <span class="s2">"last_name"</span>: <span class="s2">"Stark"</span><span class="o">}}</span> +<span class="o">{</span> <span class="s2">"title"</span> : <span class="s2">"Dancing Master"</span>, <span class="s2">"name"</span> : <span class="o">{</span> <span class="s2">"first_name"</span>: <span class="s2">"Syrio"</span>, <span class="s2">"last_name"</span>: <span class="s2">"Forel"</span><span class="o">}}</span> +</pre></div> +</div> +<p>Tajo provides two ways to create a table for this data. First is a traditional way to create tables. Here is an example.</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">EXTERNAL</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">title</span> <span class="nb">TEXT</span><span class="p">,</span> + <span class="n">name</span> <span class="n">RECORD</span> <span class="p">(</span> + <span class="n">first_name</span> <span class="nb">TEXT</span><span class="p">,</span> + <span class="n">last_name</span> <span class="nb">TEXT</span> + <span class="p">)</span> +<span class="p">)</span> <span class="k">USING</span> <span class="n">JSON</span> <span class="k">LOCATION</span> <span class="s1">'/table1/table.json'</span><span class="p">;</span> +</pre></div> +</div> +<p>With this way, you need to specify every column which they want to use. This will be a tedious work, and not appropriate for flexible JSON schema. +Second is a simpler alternative to alleviate this problem. When you create an external table of JSON format, you can simply omit the column specification as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">EXTERNAL</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="o">*</span><span class="p">)</span> <span class="k">USING</span> <span class="n">JSON</span> <span class="k">LOCATION</span> <span class="s1">'/table1/table.json'</span><span class="p">;</span> +</pre></div> +</div> +<p>No matter which way you choose, you can submit any queries on this table.</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="o">></span> <span class="k">SELECT</span> <span class="n">title</span><span class="p">,</span> <span class="n">name</span><span class="p">.</span><span class="n">last_name</span> <span class="k">from</span> <span class="n">table1</span> <span class="k">where</span> <span class="n">name</span><span class="p">.</span><span class="n">first_name</span> <span class="o">=</span> <span class="s1">'Arya'</span><span class="p">;</span> +<span class="n">title</span><span class="p">,</span><span class="n">name</span><span class="o">/</span><span class="n">last_name</span> +<span class="c1">-------------------------------</span> +<span class="n">Assassin</span><span class="p">,</span><span class="n">Stark</span> +</pre></div> +</div> +<div class="admonition warning"> +<p class="first admonition-title">Warning</p> +<p class="last">If you create a table with the second way, every column is assumed as the <code class="docutils literal"><span class="pre">TEXT</span></code> type. +So, you need to perform type casting if you want to handle them as other types.</p> +</div> +</div> +<div class="section" id="physical-properties"> +<h2>Physical Properties<a class="headerlink" href="#physical-properties" title="Permalink to this headline">¶</a></h2> +<p>Some table storage formats provide parameters for enabling or disabling features and adjusting physical parameters. +The <code class="docutils literal"><span class="pre">WITH</span></code> clause in the CREATE TABLE statement allows users to set those parameters.</p> +<p>The JSON format provides the following physical properties.</p> +<ul class="simple"> +<li><code class="docutils literal"><span class="pre">text.delimiter</span></code>: delimiter character. <code class="docutils literal"><span class="pre">|</span></code> or <code class="docutils literal"><span class="pre">\u0001</span></code> is usually used, and the default field delimiter is <code class="docutils literal"><span class="pre">|</span></code>.</li> +<li><code class="docutils literal"><span class="pre">text.null</span></code>: <code class="docutils literal"><span class="pre">NULL</span></code> character. The default <code class="docutils literal"><span class="pre">NULL</span></code> character is an empty string <code class="docutils literal"><span class="pre">''</span></code>. Hive’s default <code class="docutils literal"><span class="pre">NULL</span></code> character is <code class="docutils literal"><span class="pre">'\\N'</span></code>.</li> +<li><code class="docutils literal"><span class="pre">compression.codec</span></code>: Compression codec. You can enable compression feature and set specified compression algorithm. The compression algorithm used to compress files. The compression codec name should be the fully qualified class name inherited from <a class="reference external" href="https://hadoop.apache.org/docs/current/api/org/apache/hadoop/io/compress/CompressionCodec.html";>org.apache.hadoop.io.compress.CompressionCodec</a>. By default, compression is disabled.</li> +<li><code class="docutils literal"><span class="pre">timezone</span></code>: the time zone that the table uses for writting. When table rows are read or written, <code class="docutils literal"><span class="pre">`timestamp`</span></code> and <code class="docutils literal"><span class="pre">`time`</span></code> column values are adjusted by this timezone if it is set. Time zone can be an abbreviation form like ‘PST’ or ‘DST’. Also, it accepts an offset-based form like ‘UTC+9’ or a location-based form like ‘Asia/Seoul’.</li> +<li><code class="docutils literal"><span class="pre">text.error-tolerance.max-num</span></code>: the maximum number of permissible parsing errors. This value should be an integer value. By default, <code class="docutils literal"><span class="pre">text.error-tolerance.max-num</span></code> is <code class="docutils literal"><span class="pre">0</span></code>. According to the value, parsing errors will be handled in different ways.<ul> +<li>If <code class="docutils literal"><span class="pre">text.error-tolerance.max-num</span> <span class="pre"><</span> <span class="pre">0</span></code>, all parsing errors are ignored.</li> +<li>If <code class="docutils literal"><span class="pre">text.error-tolerance.max-num</span> <span class="pre">==</span> <span class="pre">0</span></code>, any parsing error is not allowed. If any error occurs, the query will be failed. (default)</li> +<li>If <code class="docutils literal"><span class="pre">text.error-tolerance.max-num</span> <span class="pre">></span> <span class="pre">0</span></code>, the given number of parsing errors in each task will be pemissible.</li> +</ul> +</li> +<li><code class="docutils literal"><span class="pre">text.skip.headerlines</span></code>: Number of header lines to be skipped. Some text files often have a header which has a kind of metadata(e.g.: column names), thus this option can be useful.</li> +</ul> +<p>The following example is to set a custom field delimiter, <code class="docutils literal"><span class="pre">NULL</span></code> character, and compression codec:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">id</span> <span class="nb">int</span><span class="p">,</span> + <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> + <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> + <span class="k">type</span> <span class="nb">text</span> +<span class="p">)</span> <span class="k">USING</span> <span class="n">JSON</span> <span class="k">WITH</span><span class="p">(</span><span class="s1">'text.delimiter'</span><span class="o">=</span><span class="s1">'\u0001'</span><span class="p">,</span> + <span class="s1">'text.null'</span><span class="o">=</span><span class="s1">'\\N'</span><span class="p">,</span> + <span class="s1">'compression.codec'</span><span class="o">=</span><span class="s1">'org.apache.hadoop.io.compress.SnappyCodec'</span><span class="p">);</span> +</pre></div> +</div> +<div class="admonition warning"> +<p class="first admonition-title">Warning</p> +<p class="last">Be careful when using <code class="docutils literal"><span class="pre">\n</span></code> as the field delimiter because <em>TEXT</em> format tables use <code class="docutils literal"><span class="pre">\n</span></code> as the line delimiter. +At the moment, Tajo does not provide a way to specify the line delimiter.</p> +</div> +</div> +<div class="section" id="null-value-handling-issues"> +<h2>Null Value Handling Issues<a class="headerlink" href="#null-value-handling-issues" title="Permalink to this headline">¶</a></h2> +<p>In default, <code class="docutils literal"><span class="pre">NULL</span></code> character in <em>TEXT</em> format is an empty string <code class="docutils literal"><span class="pre">''</span></code>. +In other words, an empty field is basically recognized as a <code class="docutils literal"><span class="pre">NULL</span></code> value in Tajo. +If a field domain is <code class="docutils literal"><span class="pre">TEXT</span></code>, an empty field is recognized as a string value <code class="docutils literal"><span class="pre">''</span></code> instead of <code class="docutils literal"><span class="pre">NULL</span></code> value. +Besides, You can also use your own <code class="docutils literal"><span class="pre">NULL</span></code> character by specifying a physical property <code class="docutils literal"><span class="pre">text.null</span></code>.</p> +</div> +</div> + + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="rcfile.html" class="btn btn-neutral float-right" title="RCFile"/>Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="text.html" class="btn btn-neutral" title="TEXT"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2014, Apache Tajo Team. + </p> + </div> + + <a href="https://github.com/snide/sphinx_rtd_theme";>Sphinx theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a> +</footer> + </div> + </div> + + </section> + + </div> + + + + + + <script type="text/javascript"> + var DOCUMENTATION_OPTIONS = { + URL_ROOT:'../', + VERSION:'0.11.0', + COLLAPSE_INDEX:false, + FILE_SUFFIX:'.html', + HAS_SOURCE: true + }; + </script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.StickyNav.enable(); + }); + </script> + + +</body> +</html> \ No newline at end of file
Added: tajo/site/docs/devel/table_management/orc.html URL: http://svn.apache.org/viewvc/tajo/site/docs/devel/table_management/orc.html?rev=1706955&view=auto ============================================================================== --- tajo/site/docs/devel/table_management/orc.html (added) +++ tajo/site/docs/devel/table_management/orc.html Tue Oct 6 06:45:04 2015 @@ -0,0 +1,316 @@ + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>ORC — Apache Tajo 0.11.0 documentation</title> + + + + + + + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + + + + <link rel="top" title="Apache Tajo 0.11.0 documentation" href="../index.html"/> + <link rel="up" title="File Formats" href="file_formats.html"/> + <link rel="next" title="SequenceFile" href="sequencefile.html"/> + <link rel="prev" title="Parquet" href="parquet.html"/> + + + <script src="https://cdnjs.cloudflare.com/ajax/libs/modernizr/2.6.2/modernizr.min.js";></script> + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-nav-search"> + <a href="../index.html" class="fa fa-home"> Apache Tajo</a> + <div role="search"> + <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + <ul class="current"> +<li class="toctree-l1"><a class="reference internal" href="../introduction.html">Introduction</a></li> +<li class="toctree-l1"><a class="reference internal" href="../getting_started.html">Getting Started</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#prerequisites">Prerequisites</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#dowload-and-unpack-the-source-code">Dowload and unpack the source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#build-source-code">Build source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#setting-up-a-local-tajo-cluster">Setting up a local Tajo cluster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#first-query-execution">First query execution</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../configuration/preliminary.html">Preliminary</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/cluster_setup.html">Cluster Setup</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo_master_configuration.html">Tajo Master Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/worker_configuration.html">Worker Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog_configuration.html">Catalog Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/ha_configuration.html">High Availability for TajoMaster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/service_config_defaults.html">Cluster Service Configuration Defaults</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo-site-xml.html">The tajo-site.xml File</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog-site-xml.html">The catalog-site.xml File</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tsql.html">Tajo Shell (TSQL)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../tsql/meta_command.html">Meta Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/dfs_command.html">Executing HDFS commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/variables.html">Session Variables</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/admin_command.html">Administration Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/intro.html">Introducing to TSQL</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/single_command.html">Executing a single command</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/execute_file.html">Executing Queries from Files</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/background_command.html">Executing as background process</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../sql_language.html">SQL Language</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/data_model.html">Data Model</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/ddl.html">Data Definition Language</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/insert.html">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/alter_table.html">ALTER TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/queries.html">Queries</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/joins.html">Joins</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/sql_expression.html">SQL Expressions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/predicates.html">Predicates</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../time_zone.html">Time Zone</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#server-cluster-time-zone">Server Cluster Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#table-time-zone">Table Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#client-time-zone">Client Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#time-zone-id">Time Zone ID</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#examples-of-time-zone">Examples of Time Zone</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../functions.html">Functions</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../functions.html#built-in-scalar-functions">Built-in Scalar Functions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions.html#built-in-aggregation-functions">Built-in Aggregation Functions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions.html#built-in-window-functions">Built-in Window Functions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions.html#user-defined-functions">User-defined Functions</a></li> +</ul> +</li> +<li class="toctree-l1 current"><a class="reference internal" href="../table_management.html">Table Management</a><ul class="current"> +<li class="toctree-l2"><a class="reference internal" href="table_overview.html">Overview of Tajo Tables</a></li> +<li class="toctree-l2"><a class="reference internal" href="tablespaces.html">Tablespaces</a></li> +<li class="toctree-l2 current"><a class="reference internal" href="file_formats.html">File Formats</a></li> +<li class="toctree-l2"><a class="reference internal" href="compression.html">Compression</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../table_partitioning.html">Table Partitioning</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/intro_to_partitioning.html">Introduction to Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/column_partitioning.html">Column Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/range_partitioning.html">Range Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/hash_partitioning.html">Hash Partitioning</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../storage_plugins.html">Storage Plugin</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../storage_plugins/overview.html">Storage Plugin Overview</a></li> +<li class="toctree-l2"><a class="reference internal" href="../storage_plugins/postgresql.html">PostgreSQL Storage Handler</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../index_overview.html">Index (Experimental Feature)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../index/types.html">Index Types</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/how_to_use.html">How to use index?</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/future_work.html">Future Works</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../backup_and_restore.html">Backup and Restore</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../backup_and_restore/catalog.html">Backup and Restore Catalog</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../hive_integration.html">Hive Integration</a></li> +<li class="toctree-l1"><a class="reference internal" href="../hbase_integration.html">HBase Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#create-table">CREATE TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#drop-table">DROP TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#insert-overwrite-into">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#usage">Usage</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../swift_integration.html">OpenStack Swift Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#swift-configuration">Swift configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#hadoop-configurations">Hadoop configurations</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#tajo-configuration">Tajo configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#querying-on-swift">Querying on Swift</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../jdbc_driver.html">Tajo JDBC Driver</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#how-to-get-jdbc-driver">How to get JDBC driver</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#setting-the-classpath">Setting the CLASSPATH</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#connecting-to-the-tajo-cluster-instance">Connecting to the Tajo cluster instance</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#connection-parameters">Connection Parameters</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#an-example-jdbc-client">An Example JDBC Client</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tajo_client_api.html">Tajo Client API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> +</ul> + + + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Tajo</a> + </nav> + + + + <div class="wy-nav-content"> + <div class="rst-content"> + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="../table_management.html">Table Management</a> »</li> + + <li><a href="file_formats.html">File Formats</a> »</li> + + <li>ORC</li> + <li class="wy-breadcrumbs-aside"> + + <a href="../_sources/table_management/orc.txt" rel="nofollow"> View page source</a> + + </li> + </ul> + <hr/> +</div> + <div role="main"> + + <div class="section" id="orc"> +<h1>ORC<a class="headerlink" href="#orc" title="Permalink to this headline">¶</a></h1> +<p><strong>ORC(Optimized Row Columnar)</strong> is a columnar storage format from Hive. ORC improves performance for reading, +writing, and processing data. +For more details, please refer to <a class="reference external" href="https://cwiki.apache.org/confluence/display/Hive/LanguageManual+ORC";>ORC Files</a> at Hive wiki.</p> +<div class="section" id="how-to-create-an-orc-table"> +<h2>How to Create an ORC Table?<a class="headerlink" href="#how-to-create-an-orc-table" title="Permalink to this headline">¶</a></h2> +<p>If you are not familiar with <code class="docutils literal"><span class="pre">CREATE</span> <span class="pre">TABLE</span></code> statement, please refer to Data Definition Language <a class="reference internal" href="../sql_language/ddl.html"><em>Data Definition Language</em></a>.</p> +<p>In order to specify a certain file format for your table, you need to use the <code class="docutils literal"><span class="pre">USING</span></code> clause in your <code class="docutils literal"><span class="pre">CREATE</span> <span class="pre">TABLE</span></code> +statement. Below is an example statement for creating a table using orc files.</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">id</span> <span class="nb">int</span><span class="p">,</span> + <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> + <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> + <span class="k">type</span> <span class="nb">text</span> +<span class="p">)</span> <span class="k">USING</span> <span class="n">orc</span><span class="p">;</span> +</pre></div> +</div> +</div> +<div class="section" id="physical-properties"> +<h2>Physical Properties<a class="headerlink" href="#physical-properties" title="Permalink to this headline">¶</a></h2> +<p>Some table storage formats provide parameters for enabling or disabling features and adjusting physical parameters. +The <code class="docutils literal"><span class="pre">WITH</span></code> clause in the CREATE TABLE statement allows users to set those parameters.</p> +<p>Now, ORC file provides the following physical properties.</p> +<ul class="simple"> +<li><code class="docutils literal"><span class="pre">orc.max.merge.distance</span></code>: When ORC file is read, if stripes are too closer and the distance is lower than this value, they are merged and read at once. Default is 1MB.</li> +<li><code class="docutils literal"><span class="pre">orc.stripe.size</span></code>: It decides size of each stripe. Default is 64MB.</li> +<li><code class="docutils literal"><span class="pre">orc.compression.kind</span></code>: It means the compression algorithm used to compress and write data. It should be one of <code class="docutils literal"><span class="pre">none</span></code>, <code class="docutils literal"><span class="pre">snappy</span></code>, <code class="docutils literal"><span class="pre">zlib</span></code>. Default is <code class="docutils literal"><span class="pre">none</span></code>.</li> +<li><code class="docutils literal"><span class="pre">orc.buffer.size</span></code>: It decides size of writing buffer. Default is 256KB.</li> +<li><code class="docutils literal"><span class="pre">orc.rowindex.stride</span></code>: Define the default ORC index stride in number of rows. (Stride is the number of rows an index entry represents.) Default is 10000.</li> +</ul> +</div> +<div class="section" id="compatibility-issues-with-apache-hive"> +<h2>Compatibility Issues with Apache Hiveâ¢<a class="headerlink" href="#compatibility-issues-with-apache-hive" title="Permalink to this headline">¶</a></h2> +<p>At the moment, Tajo only supports flat relational tables. +We are currently working on adding support for nested schemas and non-scalar types (<a class="reference external" href="https://issues.apache.org/jira/browse/TAJO-710";>TAJO-710</a>).</p> +</div> +</div> + + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="sequencefile.html" class="btn btn-neutral float-right" title="SequenceFile"/>Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="parquet.html" class="btn btn-neutral" title="Parquet"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2014, Apache Tajo Team. + </p> + </div> + + <a href="https://github.com/snide/sphinx_rtd_theme";>Sphinx theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a> +</footer> + </div> + </div> + + </section> + + </div> + + + + + + <script type="text/javascript"> + var DOCUMENTATION_OPTIONS = { + URL_ROOT:'../', + VERSION:'0.11.0', + COLLAPSE_INDEX:false, + FILE_SUFFIX:'.html', + HAS_SOURCE: true + }; + </script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.StickyNav.enable(); + }); + </script> + + +</body> +</html> \ No newline at end of file
