Added: tajo/site/docs/0.10.1/table_management/rcfile.html URL: http://svn.apache.org/viewvc/tajo/site/docs/0.10.1/table_management/rcfile.html?rev=1688350&view=auto ============================================================================== --- tajo/site/docs/0.10.1/table_management/rcfile.html (added) +++ tajo/site/docs/0.10.1/table_management/rcfile.html Tue Jun 30 04:16:52 2015 @@ -0,0 +1,392 @@ + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>RCFile — Apache Tajo 0.11.0 documentation</title> + + + + + + + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + + + + <link rel="top" title="Apache Tajo 0.11.0 documentation" href="../index.html"/> + <link rel="up" title="File Formats" href="file_formats.html"/> + <link rel="next" title="Parquet" href="parquet.html"/> + <link rel="prev" title="TEXT" href="text.html"/> + + + <script src="https://cdnjs.cloudflare.com/ajax/libs/modernizr/2.6.2/modernizr.min.js";></script> + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-nav-search"> + <a href="../index.html" class="fa fa-home"> Apache Tajo</a> + <div role="search"> + <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + <ul class="current"> +<li class="toctree-l1"><a class="reference internal" href="../introduction.html">Introduction</a></li> +<li class="toctree-l1"><a class="reference internal" href="../getting_started.html">Getting Started</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#prerequisites">Prerequisites</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#dowload-and-unpack-the-source-code">Dowload and unpack the source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#build-source-code">Build source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#setting-up-a-local-tajo-cluster">Setting up a local Tajo cluster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#first-query-execution">First query execution</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../configuration/preliminary.html">Preliminary</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/cluster_setup.html">Cluster Setup</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo_master_configuration.html">Tajo Master Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/worker_configuration.html">Worker Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog_configuration.html">Catalog Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/ha_configuration.html">High Availability for TajoMaster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/service_config_defaults.html">Cluster Service Configuration Defaults</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo-site-xml.html">The tajo-site.xml File</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog-site-xml.html">The catalog-site.xml File</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tsql.html">Tajo Shell (TSQL)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../tsql/meta_command.html">Meta Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/dfs_command.html">Executing HDFS commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/variables.html">Session Variables</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/admin_command.html">Administration Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/intro.html">Introducing to TSQL</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/single_command.html">Executing a single command</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/execute_file.html">Executing Queries from Files</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/background_command.html">Executing as background process</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../sql_language.html">SQL Language</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/data_model.html">Data Model</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/ddl.html">Data Definition Language</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/insert.html">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/queries.html">Queries</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/sql_expression.html">SQL Expressions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/predicates.html">Predicates</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../time_zone.html">Time Zone</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#server-cluster-time-zone">Server Cluster Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#table-time-zone">Table Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#client-time-zone">Client Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#time-zone-id">Time Zone ID</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#examples-of-time-zone">Examples of Time Zone</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../functions.html">Functions</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../functions/math_func_and_operators.html">Math Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/string_func_and_operators.html">String Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/datetime_func_and_operators.html">DateTime Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/network_func_and_operators.html">Network Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/json_func.html">JSON Functions</a></li> +</ul> +</li> +<li class="toctree-l1 current"><a class="reference internal" href="../table_management.html">Table Management</a><ul class="current"> +<li class="toctree-l2"><a class="reference internal" href="table_overview.html">Overview of Tajo Tables</a></li> +<li class="toctree-l2 current"><a class="reference internal" href="file_formats.html">File Formats</a></li> +<li class="toctree-l2"><a class="reference internal" href="compression.html">Compression</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../table_partitioning.html">Table Partitioning</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/intro_to_partitioning.html">Introduction to Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/column_partitioning.html">Column Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/range_partitioning.html">Range Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/hash_partitioning.html">Hash Partitioning</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../index_overview.html">Index (Experimental Feature)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../index/types.html">Index Types</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/how_to_use.html">How to use index?</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/future_work.html">Future Works</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../backup_and_restore.html">Backup and Restore</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../backup_and_restore/catalog.html">Backup and Restore Catalog</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../hcatalog_integration.html">HCatalog Integration</a></li> +<li class="toctree-l1"><a class="reference internal" href="../hbase_integration.html">HBase Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#create-table">CREATE TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#drop-table">DROP TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#insert-overwrite-into">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#usage">Usage</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../swift_integration.html">OpenStack Swift Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#swift-configuration">Swift configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#hadoop-configurations">Hadoop configurations</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#tajo-configuration">Tajo configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#querying-on-swift">Querying on Swift</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../jdbc_driver.html">Tajo JDBC Driver</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#how-to-get-jdbc-driver">How to get JDBC driver</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#setting-the-classpath">Setting the CLASSPATH</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#an-example-jdbc-client">An Example JDBC Client</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tajo_client_api.html">Tajo Client API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> +</ul> + + + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Tajo</a> + </nav> + + + + <div class="wy-nav-content"> + <div class="rst-content"> + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="../table_management.html">Table Management</a> »</li> + + <li><a href="file_formats.html">File Formats</a> »</li> + + <li>RCFile</li> + <li class="wy-breadcrumbs-aside"> + + <a href="../_sources/table_management/rcfile.txt" rel="nofollow"> View page source</a> + + </li> + </ul> + <hr/> +</div> + <div role="main"> + + <div class="section" id="rcfile"> +<h1>RCFile<a class="headerlink" href="#rcfile" title="Permalink to this headline">¶</a></h1> +<p>RCFile, short of Record Columnar File, are flat files consisting of binary key/value pairs, +which shares many similarities with SequenceFile.</p> +<div class="section" id="how-to-create-a-rcfile-table"> +<h2>How to Create a RCFile Table?<a class="headerlink" href="#how-to-create-a-rcfile-table" title="Permalink to this headline">¶</a></h2> +<p>If you are not familiar with the <code class="docutils literal"><span class="pre">CREATE</span> <span class="pre">TABLE</span></code> statement, please refer to the Data Definition Language <a class="reference internal" href="../sql_language/ddl.html"><em>Data Definition Language</em></a>.</p> +<p>In order to specify a certain file format for your table, you need to use the <code class="docutils literal"><span class="pre">USING</span></code> clause in your <code class="docutils literal"><span class="pre">CREATE</span> <span class="pre">TABLE</span></code> +statement. Below is an example statement for creating a table using RCFile.</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">id</span> <span class="nb">int</span><span class="p">,</span> + <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> + <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> + <span class="k">type</span> <span class="nb">text</span> +<span class="p">)</span> <span class="k">USING</span> <span class="n">RCFILE</span><span class="p">;</span> +</pre></div> +</div> +</div> +<div class="section" id="physical-properties"> +<h2>Physical Properties<a class="headerlink" href="#physical-properties" title="Permalink to this headline">¶</a></h2> +<p>Some table storage formats provide parameters for enabling or disabling features and adjusting physical parameters. +The <code class="docutils literal"><span class="pre">WITH</span></code> clause in the CREATE TABLE statement allows users to set those parameters.</p> +<p>Now, the RCFile storage type provides the following physical properties.</p> +<ul class="simple"> +<li><code class="docutils literal"><span class="pre">rcfile.serde</span></code> : custom (De)serializer class. <code class="docutils literal"><span class="pre">org.apache.tajo.storage.BinarySerializerDeserializer</span></code> is the default (de)serializer class.</li> +<li><code class="docutils literal"><span class="pre">rcfile.null</span></code> : NULL character. It is only used when a table uses <code class="docutils literal"><span class="pre">org.apache.tajo.storage.TextSerializerDeserializer</span></code>. The default NULL character is an empty string <code class="docutils literal"><span class="pre">''</span></code>. Hive’s default NULL character is <code class="docutils literal"><span class="pre">'\\N'</span></code>.</li> +<li><code class="docutils literal"><span class="pre">compression.codec</span></code> : Compression codec. You can enable compression feature and set specified compression algorithm. The compression algorithm used to compress files. The compression codec name should be the fully qualified class name inherited from <a class="reference external" href="https://hadoop.apache.org/docs/current/api/org/apache/hadoop/io/compress/CompressionCodec.html";>org.apache.hadoop.io.compress.CompressionCodec</a>. By default, compression is disabled.</li> +</ul> +<p>The following is an example for creating a table using RCFile that uses compression.</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">id</span> <span class="nb">int</span><span class="p">,</span> + <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> + <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> + <span class="k">type</span> <span class="nb">text</span> +<span class="p">)</span> <span class="k">USING</span> <span class="n">RCFILE</span> <span class="k">WITH</span> <span class="p">(</span><span class="s1">'compression.codec'</span><span class="o">=</span><span class="s1">'org.apache.hadoop.io.compress.SnappyCodec'</span><span class="p">);</span> +</pre></div> +</div> +</div> +<div class="section" id="rcfile-de-serializers"> +<h2>RCFile (De)serializers<a class="headerlink" href="#rcfile-de-serializers" title="Permalink to this headline">¶</a></h2> +<p>Tajo provides two built-in (De)serializer for RCFile:</p> +<ul class="simple"> +<li><code class="docutils literal"><span class="pre">org.apache.tajo.storage.TextSerializerDeserializer</span></code>: stores column values in a plain-text form.</li> +<li><code class="docutils literal"><span class="pre">org.apache.tajo.storage.BinarySerializerDeserializer</span></code>: stores column values in a binary file format.</li> +</ul> +<p>The RCFile format can store some metadata in the RCFile header. Tajo writes the (de)serializer class name into +the metadata header of each RCFile when the RCFile is created in Tajo.</p> +<div class="admonition note"> +<p class="first admonition-title">Note</p> +<p class="last"><code class="docutils literal"><span class="pre">org.apache.tajo.storage.BinarySerializerDeserializer</span></code> is the default (de) serializer for RCFile.</p> +</div> +</div> +<div class="section" id="compatibility-issues-with-apache-hive"> +<h2>Compatibility Issues with Apache Hiveâ¢<a class="headerlink" href="#compatibility-issues-with-apache-hive" title="Permalink to this headline">¶</a></h2> +<p>Regardless of whether the RCFiles are written by Apache Hive⢠or Apache Tajoâ¢, the files are compatible in both systems. +In other words, Tajo can process RCFiles written by Apache Hive and vice versa.</p> +<p>Since there are no metadata in RCFiles written by Hive, we need to manually specify the (de)serializer class name +by setting a physical property.</p> +<p>In Hive, there are two SerDe, and they correspond to the following (de)serializer in Tajo.</p> +<ul class="simple"> +<li><code class="docutils literal"><span class="pre">org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</span></code>: corresponds to <code class="docutils literal"><span class="pre">TextSerializerDeserializer</span></code> in Tajo.</li> +<li><code class="docutils literal"><span class="pre">org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe</span></code>: corresponds to <code class="docutils literal"><span class="pre">BinarySerializerDeserializer</span></code> in Tajo.</li> +</ul> +<p>The compatibility issue mostly occurs when a user creates an external table pointing to data of an existing table. +The following section explains two cases: 1) the case where Tajo reads RCFile written by Hive, and +2) the case where Hive reads RCFile written by Tajo.</p> +<div class="section" id="when-tajo-reads-rcfile-generated-in-hive"> +<h3>When Tajo reads RCFile generated in Hive<a class="headerlink" href="#when-tajo-reads-rcfile-generated-in-hive" title="Permalink to this headline">¶</a></h3> +<p>To create an external RCFile table generated with <code class="docutils literal"><span class="pre">ColumnarSerDe</span></code> in Hive, +you should set the physical property <code class="docutils literal"><span class="pre">rcfile.serde</span></code> in Tajo as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">EXTERNAL</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">id</span> <span class="nb">int</span><span class="p">,</span> + <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> + <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> + <span class="k">type</span> <span class="nb">text</span> +<span class="p">)</span> <span class="k">USING</span> <span class="n">RCFILE</span> <span class="k">with</span> <span class="p">(</span> <span class="s1">'rcfile.serde'</span><span class="o">=</span><span class="s1">'org.apache.tajo.storage.TextSerializerDeserializer'</span><span class="p">,</span> <span class="s1">'rcfile.null'</span><span class="o">=</span><span class="s1">'\\N'</span> <span class="p">)</span> +<span class="k">LOCATION</span> <span class="s1">'....'</span><span class="p">;</span> +</pre></div> +</div> +<p>To create an external RCFile table generated with <code class="docutils literal"><span class="pre">LazyBinaryColumnarSerDe</span></code> in Hive, +you should set the physical property <code class="docutils literal"><span class="pre">rcfile.serde</span></code> in Tajo as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">EXTERNAL</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">id</span> <span class="nb">int</span><span class="p">,</span> + <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> + <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> + <span class="k">type</span> <span class="nb">text</span> +<span class="p">)</span> <span class="k">USING</span> <span class="n">RCFILE</span> <span class="k">WITH</span> <span class="p">(</span><span class="s1">'rcfile.serde'</span> <span class="o">=</span> <span class="s1">'org.apache.tajo.storage.BinarySerializerDeserializer'</span><span class="p">)</span> +<span class="k">LOCATION</span> <span class="s1">'....'</span><span class="p">;</span> +</pre></div> +</div> +<div class="admonition note"> +<p class="first admonition-title">Note</p> +<p class="last">As we mentioned above, <code class="docutils literal"><span class="pre">BinarySerializerDeserializer</span></code> is the default (de) serializer for RCFile. +So, you can omit the <code class="docutils literal"><span class="pre">rcfile.serde</span></code> only for <code class="docutils literal"><span class="pre">org.apache.tajo.storage.BinarySerializerDeserializer</span></code>.</p> +</div> +</div> +<div class="section" id="when-hive-reads-rcfile-generated-in-tajo"> +<h3>When Hive reads RCFile generated in Tajo<a class="headerlink" href="#when-hive-reads-rcfile-generated-in-tajo" title="Permalink to this headline">¶</a></h3> +<p>To create an external RCFile table written by Tajo with <code class="docutils literal"><span class="pre">TextSerializerDeserializer</span></code>, +you should set the <code class="docutils literal"><span class="pre">SERDE</span></code> as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">id</span> <span class="nb">int</span><span class="p">,</span> + <span class="n">name</span> <span class="n">string</span><span class="p">,</span> + <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> + <span class="k">type</span> <span class="n">string</span> +<span class="p">)</span> <span class="k">ROW</span> <span class="n">FORMAT</span> <span class="n">SERDE</span> <span class="s1">'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'</span> <span class="n">STORED</span> <span class="k">AS</span> <span class="n">RCFILE</span> +<span class="k">LOCATION</span> <span class="s1">'<hdfs_location>'</span><span class="p">;</span> +</pre></div> +</div> +<p>To create an external RCFile table written by Tajo with <code class="docutils literal"><span class="pre">BinarySerializerDeserializer</span></code>, +you should set the <code class="docutils literal"><span class="pre">SERDE</span></code> as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">id</span> <span class="nb">int</span><span class="p">,</span> + <span class="n">name</span> <span class="n">string</span><span class="p">,</span> + <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> + <span class="k">type</span> <span class="n">string</span> +<span class="p">)</span> <span class="k">ROW</span> <span class="n">FORMAT</span> <span class="n">SERDE</span> <span class="s1">'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe'</span> <span class="n">STORED</span> <span class="k">AS</span> <span class="n">RCFILE</span> +<span class="k">LOCATION</span> <span class="s1">'<hdfs_location>'</span><span class="p">;</span> +</pre></div> +</div> +</div> +</div> +</div> + + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="parquet.html" class="btn btn-neutral float-right" title="Parquet"/>Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="text.html" class="btn btn-neutral" title="TEXT"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2014, Apache Tajo Team. + </p> + </div> + + <a href="https://github.com/snide/sphinx_rtd_theme";>Sphinx theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a> +</footer> + </div> + </div> + + </section> + + </div> + + + + + + <script type="text/javascript"> + var DOCUMENTATION_OPTIONS = { + URL_ROOT:'../', + VERSION:'0.11.0', + COLLAPSE_INDEX:false, + FILE_SUFFIX:'.html', + HAS_SOURCE: true + }; + </script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.StickyNav.enable(); + }); + </script> + + +</body> +</html> \ No newline at end of file
Added: tajo/site/docs/0.10.1/table_management/sequencefile.html URL: http://svn.apache.org/viewvc/tajo/site/docs/0.10.1/table_management/sequencefile.html?rev=1688350&view=auto ============================================================================== --- tajo/site/docs/0.10.1/table_management/sequencefile.html (added) +++ tajo/site/docs/0.10.1/table_management/sequencefile.html Tue Jun 30 04:16:52 2015 @@ -0,0 +1,360 @@ + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>SequenceFile — Apache Tajo 0.11.0 documentation</title> + + + + + + + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + + + + <link rel="top" title="Apache Tajo 0.11.0 documentation" href="../index.html"/> + <link rel="up" title="File Formats" href="file_formats.html"/> + <link rel="next" title="Compression" href="compression.html"/> + <link rel="prev" title="Parquet" href="parquet.html"/> + + + <script src="https://cdnjs.cloudflare.com/ajax/libs/modernizr/2.6.2/modernizr.min.js";></script> + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-nav-search"> + <a href="../index.html" class="fa fa-home"> Apache Tajo</a> + <div role="search"> + <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + <ul class="current"> +<li class="toctree-l1"><a class="reference internal" href="../introduction.html">Introduction</a></li> +<li class="toctree-l1"><a class="reference internal" href="../getting_started.html">Getting Started</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#prerequisites">Prerequisites</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#dowload-and-unpack-the-source-code">Dowload and unpack the source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#build-source-code">Build source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#setting-up-a-local-tajo-cluster">Setting up a local Tajo cluster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#first-query-execution">First query execution</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../configuration/preliminary.html">Preliminary</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/cluster_setup.html">Cluster Setup</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo_master_configuration.html">Tajo Master Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/worker_configuration.html">Worker Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog_configuration.html">Catalog Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/ha_configuration.html">High Availability for TajoMaster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/service_config_defaults.html">Cluster Service Configuration Defaults</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo-site-xml.html">The tajo-site.xml File</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog-site-xml.html">The catalog-site.xml File</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tsql.html">Tajo Shell (TSQL)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../tsql/meta_command.html">Meta Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/dfs_command.html">Executing HDFS commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/variables.html">Session Variables</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/admin_command.html">Administration Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/intro.html">Introducing to TSQL</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/single_command.html">Executing a single command</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/execute_file.html">Executing Queries from Files</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/background_command.html">Executing as background process</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../sql_language.html">SQL Language</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/data_model.html">Data Model</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/ddl.html">Data Definition Language</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/insert.html">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/queries.html">Queries</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/sql_expression.html">SQL Expressions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/predicates.html">Predicates</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../time_zone.html">Time Zone</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#server-cluster-time-zone">Server Cluster Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#table-time-zone">Table Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#client-time-zone">Client Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#time-zone-id">Time Zone ID</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#examples-of-time-zone">Examples of Time Zone</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../functions.html">Functions</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../functions/math_func_and_operators.html">Math Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/string_func_and_operators.html">String Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/datetime_func_and_operators.html">DateTime Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/network_func_and_operators.html">Network Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/json_func.html">JSON Functions</a></li> +</ul> +</li> +<li class="toctree-l1 current"><a class="reference internal" href="../table_management.html">Table Management</a><ul class="current"> +<li class="toctree-l2"><a class="reference internal" href="table_overview.html">Overview of Tajo Tables</a></li> +<li class="toctree-l2 current"><a class="reference internal" href="file_formats.html">File Formats</a></li> +<li class="toctree-l2"><a class="reference internal" href="compression.html">Compression</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../table_partitioning.html">Table Partitioning</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/intro_to_partitioning.html">Introduction to Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/column_partitioning.html">Column Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/range_partitioning.html">Range Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/hash_partitioning.html">Hash Partitioning</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../index_overview.html">Index (Experimental Feature)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../index/types.html">Index Types</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/how_to_use.html">How to use index?</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/future_work.html">Future Works</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../backup_and_restore.html">Backup and Restore</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../backup_and_restore/catalog.html">Backup and Restore Catalog</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../hcatalog_integration.html">HCatalog Integration</a></li> +<li class="toctree-l1"><a class="reference internal" href="../hbase_integration.html">HBase Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#create-table">CREATE TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#drop-table">DROP TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#insert-overwrite-into">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#usage">Usage</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../swift_integration.html">OpenStack Swift Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#swift-configuration">Swift configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#hadoop-configurations">Hadoop configurations</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#tajo-configuration">Tajo configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#querying-on-swift">Querying on Swift</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../jdbc_driver.html">Tajo JDBC Driver</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#how-to-get-jdbc-driver">How to get JDBC driver</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#setting-the-classpath">Setting the CLASSPATH</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#an-example-jdbc-client">An Example JDBC Client</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tajo_client_api.html">Tajo Client API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> +</ul> + + + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Tajo</a> + </nav> + + + + <div class="wy-nav-content"> + <div class="rst-content"> + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="../table_management.html">Table Management</a> »</li> + + <li><a href="file_formats.html">File Formats</a> »</li> + + <li>SequenceFile</li> + <li class="wy-breadcrumbs-aside"> + + <a href="../_sources/table_management/sequencefile.txt" rel="nofollow"> View page source</a> + + </li> + </ul> + <hr/> +</div> + <div role="main"> + + <div class="section" id="sequencefile"> +<h1>SequenceFile<a class="headerlink" href="#sequencefile" title="Permalink to this headline">¶</a></h1> +<div class="section" id="introduce"> +<h2>Introduce<a class="headerlink" href="#introduce" title="Permalink to this headline">¶</a></h2> +<p>SequenceFiles are flat files consisting of binary key/value pairs. +SequenceFile is basic file format which provided by Hadoop, and Hive also provides it to create a table.</p> +<p>The <code class="docutils literal"><span class="pre">USING</span> <span class="pre">sequencefile</span></code> keywords let you create a SequecneFile. Here is an example statement to create a table using <code class="docutils literal"><span class="pre">SequecneFile</span></code>:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="nb">text</span><span class="p">)</span> +<span class="k">USING</span> <span class="n">sequencefile</span><span class="p">;</span> +</pre></div> +</div> +<p>Also Tajo provides Hive compatibility for SequenceFile. The above statement can be written in Hive as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="n">string</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="n">string</span><span class="p">)</span> +<span class="n">STORED</span> <span class="k">AS</span> <span class="n">sequencefile</span><span class="p">;</span> +</pre></div> +</div> +</div> +<div class="section" id="serializerdeserializer-serde"> +<h2>SerializerDeserializer (SerDe)<a class="headerlink" href="#serializerdeserializer-serde" title="Permalink to this headline">¶</a></h2> +<p>There are two SerDe for SequenceFile as follows:</p> +<blockquote> +<div><ul class="simple"> +<li>TextSerializerDeserializer: This class can read and write data in plain text file format.</li> +<li>BinarySerializerDeserializer: This class can read and write data in binary file format.</li> +</ul> +</div></blockquote> +<p>The default is the SerDe for plain text file in Tajo. The above example statement created the table using TextSerializerDeserializer.If you want to use BinarySerializerDeserializer, you can specify it by <code class="docutils literal"><span class="pre">sequencefile.serde</span></code> keywords:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="nb">text</span><span class="p">)</span> +<span class="k">USING</span> <span class="n">sequencefile</span> <span class="k">with</span> <span class="p">(</span><span class="s1">'sequencefile.serde'</span><span class="o">=</span><span class="s1">'org.apache.tajo.storage.BinarySerializerDeserializer'</span><span class="p">)</span> +</pre></div> +</div> +<p>In Hive, the above statement can be written in Hive as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="n">string</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="n">string</span><span class="p">)</span> +<span class="k">ROW</span> <span class="n">FORMAT</span> <span class="n">SERDE</span> + <span class="s1">'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'</span> +<span class="n">STORED</span> <span class="k">AS</span> <span class="n">sequencefile</span><span class="p">;</span> +</pre></div> +</div> +</div> +<div class="section" id="writer"> +<h2>Writer<a class="headerlink" href="#writer" title="Permalink to this headline">¶</a></h2> +<p>There are three SequenceFile Writers based on the SequenceFile.CompressionType used to compress key/value pairs:</p> +<blockquote> +<div><ul class="simple"> +<li>Writer : Uncompressed records.</li> +<li>RecordCompressWriter : Record-compressed files, only compress values.</li> +<li>BlockCompressWriter : Block-compressed files, both keys & values are collected in ‘blocks’ separately and compressed. The size of the ‘block’ is configurable.</li> +</ul> +</div></blockquote> +<p>The default is Uncompressed Writer in Tajo. If you want to use RecordCompressWriter, you can specify it by <code class="docutils literal"><span class="pre">compression.type</span></code> keywords and <code class="docutils literal"><span class="pre">compression.codec</span></code> keywords:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="nb">text</span><span class="p">)</span> +<span class="k">USING</span> <span class="n">sequencefile</span> <span class="k">with</span> <span class="p">(</span><span class="s1">'compression.type'</span><span class="o">=</span><span class="s1">'RECORD'</span><span class="p">,</span><span class="s1">'compression.codec'</span><span class="o">=</span><span class="s1">'org.apache.hadoop.io.compress.SnappyCodec'</span><span class="p">)</span> +</pre></div> +</div> +<p>In hive, you need to specify settings as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="n">hive</span><span class="o">></span> <span class="k">SET</span> <span class="n">hive</span><span class="p">.</span><span class="k">exec</span><span class="p">.</span><span class="n">compress</span><span class="p">.</span><span class="k">output</span> <span class="o">=</span> <span class="k">true</span><span class="p">;</span> +<span class="n">hive</span><span class="o">></span> <span class="k">SET</span> <span class="n">mapred</span><span class="p">.</span><span class="k">output</span><span class="p">.</span><span class="n">compression</span><span class="p">.</span><span class="k">type</span> <span class="o">=</span> <span class="n">RECORD</span><span class="p">;</span> +<span class="n">hive</span><span class="o">></span> <span class="k">SET</span> <span class="n">mapred</span><span class="p">.</span><span class="k">output</span><span class="p">.</span><span class="n">compression</span><span class="p">.</span><span class="n">codec</span> <span class="o">=</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">io</span><span class="p">.</span><span class="n">compress</span><span class="p">.</span><span class="n">SnappyCodec</span><span class="p">;</span> +<span class="n">hive</span><span class="o">></span> <span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="n">string</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="n">string</span><span class="p">)</span> <span class="n">STORED</span> <span class="k">AS</span> <span class="n">sequencefile</span><span class="p">;;</span> +</pre></div> +</div> +<p>And if you want to use BlockCompressWriter, you can specify it by <code class="docutils literal"><span class="pre">compression.type</span></code> keywords and <code class="docutils literal"><span class="pre">compression.codec</span></code> keywords:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="nb">text</span><span class="p">)</span> +<span class="k">USING</span> <span class="n">sequencefile</span> <span class="k">with</span> <span class="p">(</span><span class="s1">'compression.type'</span><span class="o">=</span><span class="s1">'BLOCK'</span><span class="p">,</span><span class="s1">'compression.codec'</span><span class="o">=</span><span class="s1">'org.apache.hadoop.io.compress.SnappyCodec'</span><span class="p">)</span> +</pre></div> +</div> +<p>In hive, you need to specify settings as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="n">hive</span><span class="o">></span> <span class="k">SET</span> <span class="n">hive</span><span class="p">.</span><span class="k">exec</span><span class="p">.</span><span class="n">compress</span><span class="p">.</span><span class="k">output</span> <span class="o">=</span> <span class="k">true</span><span class="p">;</span> +<span class="n">hive</span><span class="o">></span> <span class="k">SET</span> <span class="n">mapred</span><span class="p">.</span><span class="k">output</span><span class="p">.</span><span class="n">compression</span><span class="p">.</span><span class="k">type</span> <span class="o">=</span> <span class="n">BLOCK</span><span class="p">;</span> +<span class="n">hive</span><span class="o">></span> <span class="k">SET</span> <span class="n">mapred</span><span class="p">.</span><span class="k">output</span><span class="p">.</span><span class="n">compression</span><span class="p">.</span><span class="n">codec</span> <span class="o">=</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">io</span><span class="p">.</span><span class="n">compress</span><span class="p">.</span><span class="n">SnappyCodec</span><span class="p">;</span> +<span class="n">hive</span><span class="o">></span> <span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="n">string</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="n">string</span><span class="p">)</span> <span class="n">STORED</span> <span class="k">AS</span> <span class="n">sequencefile</span><span class="p">;;</span> +</pre></div> +</div> +<p>For reference, you can use TextSerDe or BinarySerDe with compression keywords. +Here is an example statement for this case.</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="nb">text</span><span class="p">)</span> +<span class="k">USING</span> <span class="n">sequencefile</span> <span class="k">with</span> <span class="p">(</span><span class="s1">'sequencefile.serde'</span><span class="o">=</span><span class="s1">'org.apache.tajo.storage.BinarySerializerDeserializer'</span><span class="p">,</span> <span class="s1">'compression.type'</span><span class="o">=</span><span class="s1">'BLOCK'</span><span class="p">,</span><span class="s1">'compression.codec'</span><span class="o">=</span><span class="s1">'org.apache.hadoop.io.compress.SnappyCodec'</span><span class="p">)</span> +</pre></div> +</div> +<p>In hive, you need to specify settings as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="n">hive</span><span class="o">></span> <span class="k">SET</span> <span class="n">hive</span><span class="p">.</span><span class="k">exec</span><span class="p">.</span><span class="n">compress</span><span class="p">.</span><span class="k">output</span> <span class="o">=</span> <span class="k">true</span><span class="p">;</span> +<span class="n">hive</span><span class="o">></span> <span class="k">SET</span> <span class="n">mapred</span><span class="p">.</span><span class="k">output</span><span class="p">.</span><span class="n">compression</span><span class="p">.</span><span class="k">type</span> <span class="o">=</span> <span class="n">BLOCK</span><span class="p">;</span> +<span class="n">hive</span><span class="o">></span> <span class="k">SET</span> <span class="n">mapred</span><span class="p">.</span><span class="k">output</span><span class="p">.</span><span class="n">compression</span><span class="p">.</span><span class="n">codec</span> <span class="o">=</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">io</span><span class="p">.</span><span class="n">compress</span><span class="p">.</span><span class="n">SnappyCodec</span><span class="p">;</span> +<span class="n">hive</span><span class="o">></span> <span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span><span class="n">id</span> <span class="nb">int</span><span class="p">,</span> <span class="n">name</span> <span class="n">string</span><span class="p">,</span> <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> <span class="k">type</span> <span class="n">string</span><span class="p">)</span> + <span class="k">ROW</span> <span class="n">FORMAT</span> <span class="n">SERDE</span> + <span class="s1">'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'</span> + <span class="n">STORED</span> <span class="k">AS</span> <span class="n">sequencefile</span><span class="p">;;</span> +</pre></div> +</div> +</div> +</div> + + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="compression.html" class="btn btn-neutral float-right" title="Compression"/>Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="parquet.html" class="btn btn-neutral" title="Parquet"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2014, Apache Tajo Team. + </p> + </div> + + <a href="https://github.com/snide/sphinx_rtd_theme";>Sphinx theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a> +</footer> + </div> + </div> + + </section> + + </div> + + + + + + <script type="text/javascript"> + var DOCUMENTATION_OPTIONS = { + URL_ROOT:'../', + VERSION:'0.11.0', + COLLAPSE_INDEX:false, + FILE_SUFFIX:'.html', + HAS_SOURCE: true + }; + </script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.StickyNav.enable(); + }); + </script> + + +</body> +</html> \ No newline at end of file Added: tajo/site/docs/0.10.1/table_management/table_overview.html URL: http://svn.apache.org/viewvc/tajo/site/docs/0.10.1/table_management/table_overview.html?rev=1688350&view=auto ============================================================================== --- tajo/site/docs/0.10.1/table_management/table_overview.html (added) +++ tajo/site/docs/0.10.1/table_management/table_overview.html Tue Jun 30 04:16:52 2015 @@ -0,0 +1,324 @@ + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>Overview of Tajo Tables — Apache Tajo 0.11.0 documentation</title> + + + + + + + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + + + + <link rel="top" title="Apache Tajo 0.11.0 documentation" href="../index.html"/> + <link rel="up" title="Table Management" href="../table_management.html"/> + <link rel="next" title="File Formats" href="file_formats.html"/> + <link rel="prev" title="Table Management" href="../table_management.html"/> + + + <script src="https://cdnjs.cloudflare.com/ajax/libs/modernizr/2.6.2/modernizr.min.js";></script> + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-nav-search"> + <a href="../index.html" class="fa fa-home"> Apache Tajo</a> + <div role="search"> + <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + <ul class="current"> +<li class="toctree-l1"><a class="reference internal" href="../introduction.html">Introduction</a></li> +<li class="toctree-l1"><a class="reference internal" href="../getting_started.html">Getting Started</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#prerequisites">Prerequisites</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#dowload-and-unpack-the-source-code">Dowload and unpack the source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#build-source-code">Build source code</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#setting-up-a-local-tajo-cluster">Setting up a local Tajo cluster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../getting_started.html#first-query-execution">First query execution</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../configuration/preliminary.html">Preliminary</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/cluster_setup.html">Cluster Setup</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo_master_configuration.html">Tajo Master Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/worker_configuration.html">Worker Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog_configuration.html">Catalog Configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/ha_configuration.html">High Availability for TajoMaster</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/service_config_defaults.html">Cluster Service Configuration Defaults</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/tajo-site-xml.html">The tajo-site.xml File</a></li> +<li class="toctree-l2"><a class="reference internal" href="../configuration/catalog-site-xml.html">The catalog-site.xml File</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tsql.html">Tajo Shell (TSQL)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../tsql/meta_command.html">Meta Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/dfs_command.html">Executing HDFS commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/variables.html">Session Variables</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/admin_command.html">Administration Commands</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/intro.html">Introducing to TSQL</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/single_command.html">Executing a single command</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/execute_file.html">Executing Queries from Files</a></li> +<li class="toctree-l2"><a class="reference internal" href="../tsql/background_command.html">Executing as background process</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../sql_language.html">SQL Language</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/data_model.html">Data Model</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/ddl.html">Data Definition Language</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/insert.html">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/queries.html">Queries</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/sql_expression.html">SQL Expressions</a></li> +<li class="toctree-l2"><a class="reference internal" href="../sql_language/predicates.html">Predicates</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../time_zone.html">Time Zone</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#server-cluster-time-zone">Server Cluster Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#table-time-zone">Table Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#client-time-zone">Client Time Zone</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#time-zone-id">Time Zone ID</a></li> +<li class="toctree-l2"><a class="reference internal" href="../time_zone.html#examples-of-time-zone">Examples of Time Zone</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../functions.html">Functions</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../functions/math_func_and_operators.html">Math Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/string_func_and_operators.html">String Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/datetime_func_and_operators.html">DateTime Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/network_func_and_operators.html">Network Functions and Operators</a></li> +<li class="toctree-l2"><a class="reference internal" href="../functions/json_func.html">JSON Functions</a></li> +</ul> +</li> +<li class="toctree-l1 current"><a class="reference internal" href="../table_management.html">Table Management</a><ul class="current"> +<li class="toctree-l2 current"><a class="current reference internal" href="">Overview of Tajo Tables</a></li> +<li class="toctree-l2"><a class="reference internal" href="file_formats.html">File Formats</a></li> +<li class="toctree-l2"><a class="reference internal" href="compression.html">Compression</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../table_partitioning.html">Table Partitioning</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/intro_to_partitioning.html">Introduction to Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/column_partitioning.html">Column Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/range_partitioning.html">Range Partitioning</a></li> +<li class="toctree-l2"><a class="reference internal" href="../partitioning/hash_partitioning.html">Hash Partitioning</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../index_overview.html">Index (Experimental Feature)</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../index/types.html">Index Types</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/how_to_use.html">How to use index?</a></li> +<li class="toctree-l2"><a class="reference internal" href="../index/future_work.html">Future Works</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../backup_and_restore.html">Backup and Restore</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../backup_and_restore/catalog.html">Backup and Restore Catalog</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../hcatalog_integration.html">HCatalog Integration</a></li> +<li class="toctree-l1"><a class="reference internal" href="../hbase_integration.html">HBase Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#create-table">CREATE TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#drop-table">DROP TABLE</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#insert-overwrite-into">INSERT (OVERWRITE) INTO</a></li> +<li class="toctree-l2"><a class="reference internal" href="../hbase_integration.html#usage">Usage</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../swift_integration.html">OpenStack Swift Integration</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#swift-configuration">Swift configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#hadoop-configurations">Hadoop configurations</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#tajo-configuration">Tajo configuration</a></li> +<li class="toctree-l2"><a class="reference internal" href="../swift_integration.html#querying-on-swift">Querying on Swift</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../jdbc_driver.html">Tajo JDBC Driver</a><ul> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#how-to-get-jdbc-driver">How to get JDBC driver</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#setting-the-classpath">Setting the CLASSPATH</a></li> +<li class="toctree-l2"><a class="reference internal" href="../jdbc_driver.html#an-example-jdbc-client">An Example JDBC Client</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../tajo_client_api.html">Tajo Client API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> +</ul> + + + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Tajo</a> + </nav> + + + + <div class="wy-nav-content"> + <div class="rst-content"> + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="../table_management.html">Table Management</a> »</li> + + <li>Overview of Tajo Tables</li> + <li class="wy-breadcrumbs-aside"> + + <a href="../_sources/table_management/table_overview.txt" rel="nofollow"> View page source</a> + + </li> + </ul> + <hr/> +</div> + <div role="main"> + + <div class="section" id="overview-of-tajo-tables"> +<h1>Overview of Tajo Tables<a class="headerlink" href="#overview-of-tajo-tables" title="Permalink to this headline">¶</a></h1> +<div class="section" id="overview"> +<h2>Overview<a class="headerlink" href="#overview" title="Permalink to this headline">¶</a></h2> +</div> +<div class="section" id="managed-table"> +<h2>Managed Table<a class="headerlink" href="#managed-table" title="Permalink to this headline">¶</a></h2> +<div class="admonition-todo admonition" id="index-0"> +<p class="first last admonition-title">Todo</p> +</div> +</div> +<div class="section" id="external-table"> +<h2>External Table<a class="headerlink" href="#external-table" title="Permalink to this headline">¶</a></h2> +<div class="admonition-todo admonition" id="index-1"> +<p class="first last admonition-title">Todo</p> +</div> +</div> +<div class="section" id="table-properties"> +<h2>Table Properties<a class="headerlink" href="#table-properties" title="Permalink to this headline">¶</a></h2> +<p>All table formats provide parameters for enabling or disabling features and adjusting physical parameters. +The <code class="docutils literal"><span class="pre">WITH</span></code> clause in the CREATE TABLE statement allows users to set those properties.</p> +<p>The following example is to set a custom field delimiter, NULL character, and compression codec:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">id</span> <span class="nb">int</span><span class="p">,</span> + <span class="n">name</span> <span class="nb">text</span><span class="p">,</span> + <span class="n">score</span> <span class="nb">float</span><span class="p">,</span> + <span class="k">type</span> <span class="nb">text</span> +<span class="p">)</span> <span class="k">USING</span> <span class="nb">TEXT</span> <span class="k">WITH</span><span class="p">(</span><span class="s1">'text.delimiter'</span><span class="o">=</span><span class="s1">'\u0001'</span><span class="p">,</span> + <span class="s1">'text.null'</span><span class="o">=</span><span class="s1">'\\N'</span><span class="p">,</span> + <span class="s1">'compression.codec'</span><span class="o">=</span><span class="s1">'org.apache.hadoop.io.compress.SnappyCodec'</span><span class="p">);</span> +</pre></div> +</div> +<p>Each physical table layout has its own specialized properties. They will be addressed in <a class="reference internal" href="file_formats.html"><em>File Formats</em></a>.</p> +</div> +<div class="section" id="common-table-properties"> +<h2>Common Table Properties<a class="headerlink" href="#common-table-properties" title="Permalink to this headline">¶</a></h2> +<p>There are some common table properties which are used in most tables.</p> +<div class="section" id="compression"> +<h3>Compression<a class="headerlink" href="#compression" title="Permalink to this headline">¶</a></h3> +<div class="admonition-todo admonition" id="index-2"> +<p class="first last admonition-title">Todo</p> +</div> +</div> +<div class="section" id="time-zone"> +<h3>Time zone<a class="headerlink" href="#time-zone" title="Permalink to this headline">¶</a></h3> +<p>In Tajo, a table property <code class="docutils literal"><span class="pre">timezone</span></code> allows users to specify a time zone that the table uses for reading or writing.</p> +<p>You can specify a table time zone as follows:</p> +<div class="highlight-sql"><div class="highlight"><pre><span class="k">CREATE</span> <span class="k">EXTERNAL</span> <span class="k">TABLE</span> <span class="n">table1</span> <span class="p">(</span> + <span class="n">t_timestamp</span> <span class="k">TIMESTAMP</span><span class="p">,</span> + <span class="n">t_date</span> <span class="nb">DATE</span> +<span class="p">)</span> <span class="k">USING</span> <span class="nb">TEXT</span> <span class="k">WITH</span><span class="p">(</span><span class="s1">'timezone'</span><span class="o">=</span><span class="s1">'ASIA/Seoul'</span><span class="p">)</span> <span class="k">LOCATION</span> <span class="s1">'/path-to-table/'</span> +</pre></div> +</div> +<p>In order to learn time zone, please refer to <a class="reference internal" href="../time_zone.html"><em>Time Zone</em></a>.</p> +</div> +</div> +</div> + + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="file_formats.html" class="btn btn-neutral float-right" title="File Formats"/>Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="../table_management.html" class="btn btn-neutral" title="Table Management"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2014, Apache Tajo Team. + </p> + </div> + + <a href="https://github.com/snide/sphinx_rtd_theme";>Sphinx theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a> +</footer> + </div> + </div> + + </section> + + </div> + + + + + + <script type="text/javascript"> + var DOCUMENTATION_OPTIONS = { + URL_ROOT:'../', + VERSION:'0.11.0', + COLLAPSE_INDEX:false, + FILE_SUFFIX:'.html', + HAS_SOURCE: true + }; + </script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.StickyNav.enable(); + }); + </script> + + +</body> +</html> \ No newline at end of file
