http://git-wip-us.apache.org/repos/asf/orc/blob/c6e29090/docs/hive-ddl.html ---------------------------------------------------------------------- diff --git a/docs/hive-ddl.html b/docs/hive-ddl.html index 0da9356..8c360d3 100644 --- a/docs/hive-ddl.html +++ b/docs/hive-ddl.html @@ -109,12 +109,6 @@ - - - - - - <option value="/docs/index.html">Background</option> @@ -130,14 +124,6 @@ - - - - - - - - @@ -174,20 +160,6 @@ - - - - - - - - - - - - - - @@ -221,20 +193,6 @@ - - - - - - - - - - - - - - <option value="/docs/types.html">Types</option> @@ -261,12 +219,6 @@ - - - - - - <option value="/docs/indexes.html">Indexes</option> @@ -280,14 +232,6 @@ - - - - - - - - @@ -324,20 +268,6 @@ - - - - - - - - - - - - - - </optgroup> @@ -381,20 +311,6 @@ - - - - - - - - - - - - - - @@ -426,25 +342,11 @@ - - - - - - <option value="/docs/releases.html">Releases</option> - - - - - - - - </optgroup> @@ -471,12 +373,6 @@ - - - - - - <option value="/docs/hive-ddl.html">Hive DDL</option> @@ -494,14 +390,6 @@ - - - - - - - - @@ -519,12 +407,6 @@ - - - - - - <option value="/docs/hive-config.html">Hive Configuration</option> @@ -544,14 +426,6 @@ - - - - - - - - </optgroup> @@ -586,12 +460,6 @@ - - - - - - <option value="/docs/mapred.html">Using in MapRed</option> @@ -601,14 +469,6 @@ - - - - - - - - @@ -638,12 +498,6 @@ - - - - - - <option value="/docs/mapreduce.html">Using in MapReduce</option> @@ -651,14 +505,6 @@ - - - - - - - - </optgroup> @@ -679,8 +525,6 @@ - - <option value="/docs/core-java.html">Using Core Java</option> @@ -704,18 +548,6 @@ - - - - - - - - - - - - @@ -727,8 +559,6 @@ - - <option value="/docs/core-cpp.html">Using Core C++</option> @@ -754,18 +584,6 @@ - - - - - - - - - - - - </optgroup> @@ -788,8 +606,6 @@ - - <option value="/docs/cpp-tools.html">C++ Tools</option> @@ -811,18 +627,6 @@ - - - - - - - - - - - - @@ -848,12 +652,6 @@ - - - - - - <option value="/docs/java-tools.html">Java Tools</option> @@ -865,695 +663,104 @@ - - - - - - - - </optgroup> - <optgroup label="Format Specification"> - + </select> +</div> + + + <div class="unit four-fifths"> + <article> + <h1>Hive DDL</h1> + <p>ORC is well integrated into Hive, so storing your istari table as ORC +is done by adding âSTORED AS ORCâ.</p> + +<p><code>CREATE TABLE istari ( + name STRING, + color STRING +) STORED AS ORC; +</code></p> + +<p>To modify a table so that new partitions of the istari table are +stored as ORC files:</p> + +<p><code>ALTER TABLE istari SET FILEFORMAT ORC; +</code></p> + +<p>As of Hive 0.14, users can request an efficient merge of small ORC files +together by issuing a CONCATENATE command on their table or partition. The +files will be merged at the stripe level without reserialization.</p> + +<p><code>ALTER TABLE istari [PARTITION partition_spec] CONCATENATE; +</code></p> + +<p>To get information about an ORC file, use the orcfiledump command.</p> + +<p><code>% hive --orcfiledump <path_to_file> +</code></p> + +<p>As of Hive 1.1, to display the data in the ORC file, use:</p> + +<p><code>% hive --orcfiledump -d <path_to_file> +</code></p> + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/spec-intro.html">Introduction</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/file-tail.html">File Tail</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/compression.html">Compression</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/run-length.html">Run Length Encoding</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/stripes.html">Stripes</option> - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/encodings.html">Column Encodings</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/spec-index.html">Indexes</option> - - - - - - - - - - - </optgroup> - - </select> -</div> - - - <div class="unit four-fifths"> - <article> - <h1>Hive DDL</h1> - <p>ORC is well integrated into Hive, so storing your istari table as ORC -is done by adding âSTORED AS ORCâ.</p> - -<p><code>CREATE TABLE istari ( - name STRING, - color STRING -) STORED AS ORC; -</code></p> - -<p>To modify a table so that new partitions of the istari table are -stored as ORC files:</p> - -<p><code>ALTER TABLE istari SET FILEFORMAT ORC; -</code></p> - -<p>As of Hive 0.14, users can request an efficient merge of small ORC files -together by issuing a CONCATENATE command on their table or partition. The -files will be merged at the stripe level without reserialization.</p> - -<p><code>ALTER TABLE istari [PARTITION partition_spec] CONCATENATE; -</code></p> - -<p>To get information about an ORC file, use the orcfiledump command.</p> - -<p><code>% hive --orcfiledump <path_to_file> -</code></p> - -<p>As of Hive 1.1, to display the data in the ORC file, use:</p> - -<p><code>% hive --orcfiledump -d <path_to_file> -</code></p> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <div class="section-nav"> - <div class="left align-right"> - - - - <a href="/docs/releases.html" class="prev">Back</a> - - </div> - <div class="right align-left"> - - - - <a href="/docs/hive-config.html" class="next">Next</a> - - </div> - </div> - <div class="clear"></div> - - - </article> - </div> - - <div class="unit one-fifth hide-on-mobiles"> - <aside> - - <h4>Overview</h4> - - -<ul> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/index.html">Background</a></li> - - - - - - - - - - - - - - <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/types.html">Types</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/indexes.html">Indexes</a></li> - - - - - - - - - - - - <li class=""><a href="/docs/acid.html">ACID support</a></li> - - - -</ul> - - - <h4>Installing</h4> - - -<ul> - - - - - - - - - - - - - - <li class=""><a href="/docs/building.html">Building ORC</a></li> - - - - - - - - - - - - - - - - - - - + - - + - - + - - + - - + - + <div class="section-nav"> + <div class="left align-right"> + + + + <a href="/docs/releases.html" class="prev">Back</a> + + </div> + <div class="right align-left"> + + + + <a href="/docs/hive-config.html" class="next">Next</a> + + </div> + </div> + <div class="clear"></div> - <li class=""><a href="/docs/releases.html">Releases</a></li> - + </article> + </div> -</ul> - + <div class="unit one-fifth hide-on-mobiles"> + <aside> - <h4>Using in Hive</h4> + <h4>Overview</h4> <ul> @@ -1582,11 +789,7 @@ files will be merged at the stripe level without reserialization.</p> - - - - - <li class="current"><a href="/docs/hive-ddl.html">Hive DDL</a></li> + <li class=""><a href="/docs/index.html">Background</a></li> @@ -1600,34 +803,10 @@ files will be merged at the stripe level without reserialization.</p> - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li> + <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li> -</ul> - - - <h4>Using in MapReduce</h4> - - -<ul> - @@ -1664,7 +843,7 @@ files will be merged at the stripe level without reserialization.</p> - <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li> + <li class=""><a href="/docs/types.html">Types</a></li> @@ -1694,49 +873,7 @@ files will be merged at the stripe level without reserialization.</p> - - - - - - - - - - - - - <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li> - - - -</ul> - - - <h4>Using ORC Core</h4> - - -<ul> - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/core-java.html">Using Core Java</a></li> + <li class=""><a href="/docs/indexes.html">Indexes</a></li> @@ -1748,22 +885,14 @@ files will be merged at the stripe level without reserialization.</p> - - - - - - - - - <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li> + <li class=""><a href="/docs/acid.html">ACID support</a></li> </ul> - <h4>Tools</h4> + <h4>Installing</h4> <ul> @@ -1780,15 +909,7 @@ files will be merged at the stripe level without reserialization.</p> - - - - - - - - - <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li> + <li class=""><a href="/docs/building.html">Building ORC</a></li> @@ -1826,14 +947,14 @@ files will be merged at the stripe level without reserialization.</p> - <li class=""><a href="/docs/java-tools.html">Java Tools</a></li> + <li class=""><a href="/docs/releases.html">Releases</a></li> </ul> - <h4>Format Specification</h4> + <h4>Using in Hive</h4> <ul> @@ -1860,31 +981,7 @@ files will be merged at the stripe level without reserialization.</p> - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/spec-intro.html">Introduction</a></li> + <li class="current"><a href="/docs/hive-ddl.html">Hive DDL</a></li> @@ -1908,31 +1005,17 @@ files will be merged at the stripe level without reserialization.</p> - - - - - <li class=""><a href="/docs/file-tail.html">File Tail</a></li> + <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li> - - - - - +</ul> - - - - - - + <h4>Using in MapReduce</h4> - <li class=""><a href="/docs/compression.html">Compression</a></li> - +<ul> @@ -1964,19 +1047,7 @@ files will be merged at the stripe level without reserialization.</p> - - - - - - - - - - - - - <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li> + <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li> @@ -2012,13 +1083,25 @@ files will be merged at the stripe level without reserialization.</p> - + <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li> + + + +</ul> + - + <h4>Using ORC Core</h4> + +<ul> + + + + + @@ -2028,7 +1111,7 @@ files will be merged at the stripe level without reserialization.</p> - <li class=""><a href="/docs/stripes.html">Stripes</a></li> + <li class=""><a href="/docs/core-java.html">Using Core Java</a></li> @@ -2046,17 +1129,17 @@ files will be merged at the stripe level without reserialization.</p> - - - - - + <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li> + + + +</ul> + - + <h4>Tools</h4> - <li class=""><a href="/docs/encodings.html">Column Encodings</a></li> - +<ul> @@ -2076,11 +1159,17 @@ files will be merged at the stripe level without reserialization.</p> + <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li> + + + - + + + @@ -2102,7 +1191,7 @@ files will be merged at the stripe level without reserialization.</p> - <li class=""><a href="/docs/spec-index.html">Indexes</a></li> + <li class=""><a href="/docs/java-tools.html">Java Tools</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/c6e29090/docs/index.html ---------------------------------------------------------------------- diff --git a/docs/index.html b/docs/index.html index 6014e66..0d344dc 100644 --- a/docs/index.html +++ b/docs/index.html @@ -109,12 +109,6 @@ - - - - - - <option value="/docs/index.html">Background</option> @@ -130,14 +124,6 @@ - - - - - - - - @@ -174,9 +160,9 @@ - + - + @@ -188,9 +174,9 @@ - + - + @@ -207,6 +193,12 @@ + <option value="/docs/types.html">Types</option> + + + + + @@ -227,6 +219,8 @@ + <option value="/docs/indexes.html">Indexes</option> + @@ -235,7 +229,7 @@ - <option value="/docs/types.html">Types</option> + @@ -243,7 +237,7 @@ - + <option value="/docs/acid.html">ACID support</option> @@ -267,8 +261,6 @@ - <option value="/docs/indexes.html">Indexes</option> - @@ -276,24 +268,35 @@ + + + </optgroup> + <optgroup label="Installing"> + + + + + + <option value="/docs/building.html">Building ORC</option> + - + - + - <option value="/docs/acid.html">ACID support</option> + @@ -308,6 +311,10 @@ + + + + @@ -335,6 +342,8 @@ + <option value="/docs/releases.html">Releases</option> + @@ -342,7 +351,7 @@ </optgroup> - <optgroup label="Installing"> + <optgroup label="Using in Hive"> @@ -354,8 +363,6 @@ - <option value="/docs/building.html">Building ORC</option> - @@ -366,7 +373,7 @@ - + <option value="/docs/hive-ddl.html">Hive DDL</option> @@ -383,7 +390,9 @@ - + + + @@ -395,9 +404,11 @@ - + - + + <option value="/docs/hive-config.html">Hive Configuration</option> + @@ -415,7 +426,16 @@ + + + </optgroup> + <optgroup label="Using in MapReduce"> + + + + + @@ -432,7 +452,7 @@ - <option value="/docs/releases.html">Releases</option> + @@ -440,18 +460,15 @@ + <option value="/docs/mapred.html">Using in MapRed</option> + - - - </optgroup> - <optgroup label="Using in Hive"> - - + @@ -477,20 +494,27 @@ - <option value="/docs/hive-ddl.html">Hive DDL</option> - - + <option value="/docs/mapreduce.html">Using in MapReduce</option> + + + </optgroup> + <optgroup label="Using ORC Core"> + + + + + @@ -501,10 +525,12 @@ + <option value="/docs/core-java.html">Using Core Java</option> + - + - + @@ -522,13 +548,19 @@ + + + + - <option value="/docs/hive-config.html">Hive Configuration</option> + + <option value="/docs/core-cpp.html">Using Core C++</option> + @@ -556,7 +588,7 @@ </optgroup> - <optgroup label="Using in MapReduce"> + <optgroup label="Tools"> @@ -574,7 +606,7 @@ - + <option value="/docs/cpp-tools.html">C++ Tools</option> @@ -592,14 +624,12 @@ - <option value="/docs/mapred.html">Using in MapRed</option> - - + - + @@ -609,10 +639,6 @@ - - - - @@ -626,7 +652,7 @@ - + <option value="/docs/java-tools.html">Java Tools</option> @@ -637,14 +663,94 @@ + + + </optgroup> + </select> +</div> + + + <div class="unit four-fifths"> + <article> + <h1>Background</h1> + <p>Back in January 2013, we created ORC files as part of the initiative +to massively speed up Apache Hive and improve the storage efficiency +of data stored in Apache Hadoop. The focus was on enabling high speed +processing and reducing file sizes.</p> + +<p>ORC is a self-describing type-aware columnar file format designed for +Hadoop workloads. It is optimized for large streaming reads, but with +integrated support for finding required rows quickly. Storing data in +a columnar format lets the reader read, decompress, and process only +the values that are required for the current query. Because ORC files +are type-aware, the writer chooses the most appropriate encoding for +the type and builds an internal index as the file is written.</p> + +<p>Predicate pushdown uses those indexes to determine which stripes in a +file need to be read for a particular query and the row indexes can +narrow the search to a particular set of 10,000 rows. ORC supports the +complete set of types in Hive, including the complex types: structs, +lists, maps, and unions.</p> + +<p>Many large Hadoop users have adopted ORC. For instance, Facebook uses +ORC to <a href="https://s.apache.org/fb-scaling-300-pb";>save tens of petabytes</a> +in their data warehouse and demonstrated that ORC is <a href="https://s.apache.org/presto-orc";>significantly +faster</a> than RC File or Parquet. Yahoo +uses ORC to store their production data and has released some of their +<a href="https://s.apache.org/yahoo-orc";>benchmark results</a>.</p> + +<p>ORC files are divided in to <em>stripes</em> that are roughly 64MB by +default. The stripes in a file are independent of each other and form +the natural unit of distributed work. Within each stripe, the columns +are separated from each other so the reader can read just the columns +that are required.</p> + + + + + + + + + <div class="section-nav"> + <div class="left align-right"> + + <span class="prev disabled">Back</span> + + </div> + <div class="right align-left"> + + + + <a href="/docs/adopters.html" class="next">Next</a> + + </div> + </div> + <div class="clear"></div> + + + </article> + </div> + + <div class="unit one-fifth hide-on-mobiles"> + <aside> + + <h4>Overview</h4> + +<ul> + + + + + - <option value="/docs/mapreduce.html">Using in MapReduce</option> + @@ -659,11 +765,8 @@ - - - </optgroup> - <optgroup label="Using ORC Core"> + <li class="current"><a href="/docs/index.html">Background</a></li> @@ -672,19 +775,21 @@ - + + <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li> + + + - + - <option value="/docs/core-java.html">Using Core Java</option> - - + @@ -715,22 +820,20 @@ - + <li class=""><a href="/docs/types.html">Types</a></li> + - - + - + - <option value="/docs/core-cpp.html">Using Core C++</option> - @@ -747,860 +850,26 @@ + <li class=""><a href="/docs/indexes.html">Indexes</a></li> + + + - + - + - - - - - - - - - - - - - - </optgroup> - - <optgroup label="Tools"> - - - - - - - - - - - - - - - - - - - - <option value="/docs/cpp-tools.html">C++ Tools</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/java-tools.html">Java Tools</option> - - - - - - - - - - - - - - - - - - - - - </optgroup> - - <optgroup label="Format Specification"> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/spec-intro.html">Introduction</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/file-tail.html">File Tail</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/compression.html">Compression</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/run-length.html">Run Length Encoding</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/stripes.html">Stripes</option> - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/encodings.html">Column Encodings</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/spec-index.html">Indexes</option> - - - - - - - - - - - </optgroup> - - </select> -</div> - - - <div class="unit four-fifths"> - <article> - <h1>Background</h1> - <p>Back in January 2013, we created ORC files as part of the initiative -to massively speed up Apache Hive and improve the storage efficiency -of data stored in Apache Hadoop. The focus was on enabling high speed -processing and reducing file sizes.</p> - -<p>ORC is a self-describing type-aware columnar file format designed for -Hadoop workloads. It is optimized for large streaming reads, but with -integrated support for finding required rows quickly. Storing data in -a columnar format lets the reader read, decompress, and process only -the values that are required for the current query. Because ORC files -are type-aware, the writer chooses the most appropriate encoding for -the type and builds an internal index as the file is written.</p> - -<p>Predicate pushdown uses those indexes to determine which stripes in a -file need to be read for a particular query and the row indexes can -narrow the search to a particular set of 10,000 rows. ORC supports the -complete set of types in Hive, including the complex types: structs, -lists, maps, and unions.</p> - -<p>Many large Hadoop users have adopted ORC. For instance, Facebook uses -ORC to <a href="https://s.apache.org/fb-scaling-300-pb";>save tens of petabytes</a> -in their data warehouse and demonstrated that ORC is <a href="https://s.apache.org/presto-orc";>significantly -faster</a> than RC File or Parquet. Yahoo -uses ORC to store their production data and has released some of their -<a href="https://s.apache.org/yahoo-orc";>benchmark results</a>.</p> - -<p>ORC files are divided in to <em>stripes</em> that are roughly 64MB by -default. The stripes in a file are independent of each other and form -the natural unit of distributed work. Within each stripe, the columns -are separated from each other so the reader can read just the columns -that are required.</p> - - - - - - - - - - <div class="section-nav"> - <div class="left align-right"> - - <span class="prev disabled">Back</span> - - </div> - <div class="right align-left"> - - - - <a href="/docs/adopters.html" class="next">Next</a> - - </div> - </div> - <div class="clear"></div> - - - </article> - </div> - - <div class="unit one-fifth hide-on-mobiles"> - <aside> - - <h4>Overview</h4> - - -<ul> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class="current"><a href="/docs/index.html">Background</a></li> - - - - - - - - - - - - - - <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/types.html">Types</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/indexes.html">Indexes</a></li> - - - - - - - - - - - - <li class=""><a href="/docs/acid.html">ACID support</a></li> - - - -</ul> - - - <h4>Installing</h4> - - -<ul> - - - - - - - - - - - - - - <li class=""><a href="/docs/building.html">Building ORC</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/releases.html">Releases</a></li> - - - -</ul> - - - <h4>Using in Hive</h4> - - -<ul> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li> + <li class=""><a href="/docs/acid.html">ACID support</a></li> </ul> - <h4>Using in MapReduce</h4> + <h4>Installing</h4> <ul> @@ -1617,31 +886,7 @@ that are required.</p> - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li> + <li class=""><a href="/docs/building.html">Building ORC</a></li> @@ -1679,18 +924,14 @@ that are required.</p> - - - - - <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li> + <li class=""><a href="/docs/releases.html">Releases</a></li> </ul> - <h4>Using ORC Core</h4> + <h4>Using in Hive</h4> <ul> @@ -1713,59 +954,11 @@ that are required.</p> - <li class=""><a href="/docs/core-java.html">Using Core Java</a></li> - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li> - - - -</ul> - - - <h4>Tools</h4> - - -<ul> - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li> + <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li> @@ -1789,28 +982,14 @@ that are required.</p> - - - - - - - - - - - - - - - <li class=""><a href="/docs/java-tools.html">Java Tools</a></li> + <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li> </ul> - <h4>Format Specification</h4> + <h4>Using in MapReduce</h4> <ul> @@ -1845,23 +1024,7 @@ that are required.</p> - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/spec-intro.html">Introduction</a></li> + <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li> @@ -1889,16 +1052,6 @@ that are required.</p> - <li class=""><a href="/docs/file-tail.html">File Tail</a></li> - - - - - - - - - @@ -1907,42 +1060,24 @@ that are required.</p> - <li class=""><a href="/docs/compression.html">Compression</a></li> + <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li> - - - - - +</ul> - - - - - - - - - - - - - - - - - - + <h4>Using ORC Core</h4> + +<ul> + - + - + @@ -1953,7 +1088,7 @@ that are required.</p> - <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li> + <li class=""><a href="/docs/core-java.html">Using Core Java</a></li> @@ -1971,54 +1106,24 @@ that are required.</p> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/stripes.html">Stripes</a></li> + <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li> - +</ul> - + <h4>Tools</h4> + + +<ul> + - + @@ -2031,7 +1136,7 @@ that are required.</p> - <li class=""><a href="/docs/encodings.html">Column Encodings</a></li> + <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li> @@ -2063,23 +1168,7 @@ that are required.</p> - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/spec-index.html">Indexes</a></li> + <li class=""><a href="/docs/java-tools.html">Java Tools</a></li> http://git-wip-us.apache.org/repos/asf/orc/blob/c6e29090/docs/indexes.html ---------------------------------------------------------------------- diff --git a/docs/indexes.html b/docs/indexes.html index 5654a47..0a81f43 100644 --- a/docs/indexes.html +++ b/docs/indexes.html @@ -109,12 +109,6 @@ - - - - - - <option value="/docs/index.html">Background</option> @@ -130,14 +124,6 @@ - - - - - - - - @@ -174,20 +160,6 @@ - - - - - - - - - - - - - - @@ -221,20 +193,6 @@ - - - - - - - - - - - - - - <option value="/docs/types.html">Types</option> @@ -261,12 +219,6 @@ - - - - - - <option value="/docs/indexes.html">Indexes</option> @@ -280,14 +232,6 @@ - - - - - - - - @@ -324,20 +268,6 @@ - - - - - - - - - - - - - - </optgroup> @@ -381,20 +311,6 @@ - - - - - - - - - - - - - - @@ -426,25 +342,11 @@ - - - - - - <option value="/docs/releases.html">Releases</option> - - - - - - - - </optgroup> @@ -471,12 +373,6 @@ - - - - - - <option value="/docs/hive-ddl.html">Hive DDL</option> @@ -494,14 +390,6 @@ - - - - - - - - @@ -519,12 +407,6 @@ - - - - - - <option value="/docs/hive-config.html">Hive Configuration</option> @@ -544,14 +426,6 @@ - - - - - - - - </optgroup> @@ -586,12 +460,6 @@ - - - - - - <option value="/docs/mapred.html">Using in MapRed</option> @@ -601,14 +469,6 @@ - - - - - - - - @@ -638,12 +498,6 @@ - - - - - - <option value="/docs/mapreduce.html">Using in MapReduce</option> @@ -651,14 +505,6 @@ - - - - - - - - </optgroup> @@ -679,8 +525,6 @@ - - <option value="/docs/core-java.html">Using Core Java</option> @@ -704,18 +548,6 @@ - - - - - - - - - - - - @@ -727,8 +559,6 @@ - - <option value="/docs/core-cpp.html">Using Core C++</option> @@ -754,18 +584,6 @@ - - - - - - - - - - - - </optgroup> @@ -788,8 +606,6 @@ - - <option value="/docs/cpp-tools.html">C++ Tools</option> @@ -811,18 +627,6 @@ - - - - - - - - - - - - @@ -848,12 +652,6 @@ - - - - - - <option value="/docs/java-tools.html">Java Tools</option> @@ -865,680 +663,89 @@ - - - - - - - - </optgroup> - <optgroup label="Format Specification"> - + </select> +</div> + + + <div class="unit four-fifths"> + <article> + <h1>Indexes</h1> + <p>ORC provides three level of indexes within each file:</p> + +<ul> + <li>file level - statistics about the values in each column across the entire +file</li> + <li>stripe level - statistics about the values in each column for each stripe</li> + <li>row level - statistics about the values in each column for each set of +10,000 rows within a stripe</li> +</ul> + +<p>The file and stripe level column statistics are in the file footer so +that they are easy to access to determine if the rest of the file +needs to be read at all. Row level indexes include both the column +statistics for each row group and the position for seeking to the +start of the row group.</p> + +<p>Column statistics always contain the count of values and whether there +are null values present. Most other primitive types include the +minimum and maximum values and for numeric types the sum. As of Hive +1.2, the indexes can include bloom filters, which provide a much more +selective filter.</p> + +<p>The indexes at all levels are used by the reader using Search +ARGuments or SARGs, which are simplified expressions that restrict the +rows that are of interest. For example, if a query was looking for +people older than 100 years old, the SARG would be âage > 100â and +only files, stripes, or row groups that had people over 100 years old +would be read.</p> + + + + + + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/spec-intro.html">Introduction</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/file-tail.html">File Tail</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/compression.html">Compression</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/run-length.html">Run Length Encoding</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/stripes.html">Stripes</option> - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/encodings.html">Column Encodings</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/spec-index.html">Indexes</option> - - - - - - - - - - - </optgroup> - - </select> -</div> - - - <div class="unit four-fifths"> - <article> - <h1>Indexes</h1> - <p>ORC provides three level of indexes within each file:</p> - -<ul> - <li>file level - statistics about the values in each column across the entire -file</li> - <li>stripe level - statistics about the values in each column for each stripe</li> - <li>row level - statistics about the values in each column for each set of -10,000 rows within a stripe</li> -</ul> - -<p>The file and stripe level column statistics are in the file footer so -that they are easy to access to determine if the rest of the file -needs to be read at all. Row level indexes include both the column -statistics for each row group and the position for seeking to the -start of the row group.</p> - -<p>Column statistics always contain the count of values and whether there -are null values present. Most other primitive types include the -minimum and maximum values and for numeric types the sum. As of Hive -1.2, the indexes can include bloom filters, which provide a much more -selective filter.</p> - -<p>The indexes at all levels are used by the reader using Search -ARGuments or SARGs, which are simplified expressions that restrict the -rows that are of interest. For example, if a query was looking for -people older than 100 years old, the SARG would be âage > 100â and -only files, stripes, or row groups that had people over 100 years old -would be read.</p> - - - - - - - - - - - - - - - - - - - <div class="section-nav"> - <div class="left align-right"> - - - - <a href="/docs/types.html" class="prev">Back</a> - - </div> - <div class="right align-left"> - - - - <a href="/docs/acid.html" class="next">Next</a> - - </div> - </div> - <div class="clear"></div> - - - </article> - </div> - - <div class="unit one-fifth hide-on-mobiles"> - <aside> - - <h4>Overview</h4> - - -<ul> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/index.html">Background</a></li> - - - - - - - - - - - - - - <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/types.html">Types</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class="current"><a href="/docs/indexes.html">Indexes</a></li> - - - - - - - - - - - - <li class=""><a href="/docs/acid.html">ACID support</a></li> - - - -</ul> - - - <h4>Installing</h4> - - -<ul> - - - - - - - - - - - - - - <li class=""><a href="/docs/building.html">Building ORC</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + <div class="section-nav"> + <div class="left align-right"> + + + + <a href="/docs/types.html" class="prev">Back</a> + + </div> + <div class="right align-left"> + + + + <a href="/docs/acid.html" class="next">Next</a> + + </div> + </div> + <div class="clear"></div> - <li class=""><a href="/docs/releases.html">Releases</a></li> - + </article> + </div> -</ul> - + <div class="unit one-fifth hide-on-mobiles"> + <aside> - <h4>Using in Hive</h4> + <h4>Overview</h4> <ul> @@ -1567,11 +774,7 @@ would be read.</p> - - - - - <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li> + <li class=""><a href="/docs/index.html">Background</a></li> @@ -1585,34 +788,10 @@ would be read.</p> - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li> + <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li> -</ul> - - - <h4>Using in MapReduce</h4> - - -<ul> - @@ -1649,7 +828,7 @@ would be read.</p> - <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li> + <li class=""><a href="/docs/types.html">Types</a></li> @@ -1679,49 +858,7 @@ would be read.</p> - - - - - - - - - - - - - <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li> - - - -</ul> - - - <h4>Using ORC Core</h4> - - -<ul> - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/core-java.html">Using Core Java</a></li> + <li class="current"><a href="/docs/indexes.html">Indexes</a></li> @@ -1733,22 +870,14 @@ would be read.</p> - - - - - - - - - <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li> + <li class=""><a href="/docs/acid.html">ACID support</a></li> </ul> - <h4>Tools</h4> + <h4>Installing</h4> <ul> @@ -1765,15 +894,7 @@ would be read.</p> - - - - - - - - - <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li> + <li class=""><a href="/docs/building.html">Building ORC</a></li> @@ -1811,14 +932,14 @@ would be read.</p> - <li class=""><a href="/docs/java-tools.html">Java Tools</a></li> + <li class=""><a href="/docs/releases.html">Releases</a></li> </ul> - <h4>Format Specification</h4> + <h4>Using in Hive</h4> <ul> @@ -1845,31 +966,7 @@ would be read.</p> - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/spec-intro.html">Introduction</a></li> + <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li> @@ -1893,31 +990,17 @@ would be read.</p> - - - - - <li class=""><a href="/docs/file-tail.html">File Tail</a></li> + <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li> - - - - - +</ul> - - - - - - + <h4>Using in MapReduce</h4> - <li class=""><a href="/docs/compression.html">Compression</a></li> - +<ul> @@ -1949,19 +1032,7 @@ would be read.</p> - - - - - - - - - - - - - <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li> + <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li> @@ -1997,13 +1068,25 @@ would be read.</p> - + <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li> + + + +</ul> + - + <h4>Using ORC Core</h4> + +<ul> + + + + + @@ -2013,7 +1096,7 @@ would be read.</p> - <li class=""><a href="/docs/stripes.html">Stripes</a></li> + <li class=""><a href="/docs/core-java.html">Using Core Java</a></li> @@ -2031,17 +1114,17 @@ would be read.</p> - - - - - + <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li> + + + +</ul> + - + <h4>Tools</h4> - <li class=""><a href="/docs/encodings.html">Column Encodings</a></li> - +<ul> @@ -2061,11 +1144,17 @@ would be read.</p> + <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li> + + + - + + + @@ -2087,7 +1176,7 @@ would be read.</p> - <li class=""><a href="/docs/spec-index.html">Indexes</a></li> + <li class=""><a href="/docs/java-tools.html">Java Tools</a></li> http://git-wip-us.apache.org/repos/asf/orc/blob/c6e29090/docs/java-tools.html ---------------------------------------------------------------------- diff --git a/docs/java-tools.html b/docs/java-tools.html index 7d38769..25efb43 100644 --- a/docs/java-tools.html +++ b/docs/java-tools.html @@ -109,12 +109,6 @@ - - - - - - <option value="/docs/index.html">Background</option> @@ -130,14 +124,6 @@ - - - - - - - - @@ -174,20 +160,6 @@ - - - - - - - - - - - - - - @@ -221,20 +193,6 @@ - - - - - - - - - - - - - - <option value="/docs/types.html">Types</option> @@ -261,12 +219,6 @@ - - - - - - <option value="/docs/indexes.html">Indexes</option> @@ -280,14 +232,6 @@ - - - - - - - - @@ -324,20 +268,6 @@ - - - - - - - - - - - - - - </optgroup> @@ -381,20 +311,6 @@ - - - - - - - - - - - - - - @@ -426,25 +342,11 @@ - - - - - - <option value="/docs/releases.html">Releases</option> - - - - - - - - </optgroup> @@ -471,12 +373,6 @@ - - - - - - <option value="/docs/hive-ddl.html">Hive DDL</option> @@ -494,14 +390,6 @@ - - - - - - - - @@ -519,12 +407,6 @@ - - - - - - <option value="/docs/hive-config.html">Hive Configuration</option> @@ -544,14 +426,6 @@ - - - - - - - - </optgroup> @@ -586,12 +460,6 @@ - - - - - - <option value="/docs/mapred.html">Using in MapRed</option> @@ -601,14 +469,6 @@ - - - - - - - - @@ -638,12 +498,6 @@ - - - - - - <option value="/docs/mapreduce.html">Using in MapReduce</option> @@ -651,14 +505,6 @@ - - - - - - - - </optgroup> @@ -679,8 +525,6 @@ - - <option value="/docs/core-java.html">Using Core Java</option> @@ -704,18 +548,6 @@ - - - - - - - - - - - - @@ -727,8 +559,6 @@ - - <option value="/docs/core-cpp.html">Using Core C++</option> @@ -754,18 +584,6 @@ - - - - - - - - - - - - </optgroup> @@ -788,8 +606,6 @@ - - <option value="/docs/cpp-tools.html">C++ Tools</option> @@ -811,18 +627,6 @@ - - - - - - - - - - - - @@ -848,12 +652,6 @@ - - - - - - <option value="/docs/java-tools.html">Java Tools</option> @@ -865,992 +663,329 @@ - - - - - - - - </optgroup> - <optgroup label="Format Specification"> - + </select> +</div> - + <div class="unit four-fifths"> + <article> + <h1>Java Tools</h1> + <p>In addition to the C++ tools, there is an ORC tools jar that packages +several useful utilities and the necessary Java dependencies +(including Hadoop) into a single package. The Java ORC tool jar +supports both the local file system and HDFS.</p> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/spec-intro.html">Introduction</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/file-tail.html">File Tail</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/compression.html">Compression</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/run-length.html">Run Length Encoding</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/stripes.html">Stripes</option> - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/encodings.html">Column Encodings</option> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <option value="/docs/spec-index.html">Indexes</option> - - - - - - - - - - - </optgroup> - - </select> -</div> - - - <div class="unit four-fifths"> - <article> - <h1>Java Tools</h1> - <p>In addition to the C++ tools, there is an ORC tools jar that packages -several useful utilities and the necessary Java dependencies -(including Hadoop) into a single package. The Java ORC tool jar -supports both the local file system and HDFS.</p> - -<p>The subcommands for the tools are:</p> - -<ul> - <li>meta - print the metadata of an ORC file</li> - <li>data - print the data of an ORC file</li> - <li>scan (since ORC 1.3) - scan the data for benchmarking</li> - <li>convert (since ORC 1.4) - convert JSON files to ORC</li> - <li>json-schema (since ORC 1.4) - determine the schema of JSON documents</li> -</ul> - -<p>The command line looks like:</p> - -<pre><code class="language-shell">% java -jar orc-tools-X.Y.Z-uber.jar <sub-command> <args> -</code></pre> - -<h2 id="java-meta">Java Meta</h2> - -<p>The meta command prints the metadata about the given ORC file and is -equivalent to the Hive ORC File Dump command.</p> - -<dl> - <dt>-j</dt> - <dd>format the output in JSON</dd> - <dt>-p</dt> - <dd>pretty print the output</dd> - <dt>-t</dt> - <dd>print the timezone of the writer</dd> - <dt>ârowindex</dt> - <dd>print the row indexes for the comma separated list of column ids</dd> - <dt>ârecover</dt> - <dd>skip over corrupted values in the ORC file</dd> - <dt>âskip-dump</dt> - <dd>skip dumping the metadata</dd> - <dt>âbackup-path</dt> - <dd>when used with ârecover specifies the path where the recovered file is written</dd> -</dl> - -<p>An example of the output is given below:</p> - -<pre><code class="language-shell">% java -jar orc-tools-X.Y.Z-uber.jar meta examples/TestOrcFile.test1.orc -Processing data file examples/TestOrcFile.test1.orc [length: 1711] -Structure for examples/TestOrcFile.test1.orc -File Version: 0.12 with HIVE_8732 -Rows: 2 -Compression: ZLIB -Compression size: 10000 -Type: struct<boolean1:boolean,byte1:tinyint,short1:smallint,int1:int, -long1:bigint,float1:float,double1:double,bytes1:binary,string1:string, -middle:struct<list:array<struct<int1:int,string1:string>>>,list:array< -struct<int1:int,string1:string>>,map:map<string,struct<int1:int,string1: -string>>> - -Stripe Statistics: - Stripe 1: - Column 0: count: 2 hasNull: false - Column 1: count: 2 hasNull: false true: 1 - Column 2: count: 2 hasNull: false min: 1 max: 100 sum: 101 - Column 3: count: 2 hasNull: false min: 1024 max: 2048 sum: 3072 - Column 4: count: 2 hasNull: false min: 65536 max: 65536 sum: 131072 - Column 5: count: 2 hasNull: false min: 9223372036854775807 max: 9223372036854775807 - Column 6: count: 2 hasNull: false min: 1.0 max: 2.0 sum: 3.0 - Column 7: count: 2 hasNull: false min: -15.0 max: -5.0 sum: -20.0 - Column 8: count: 2 hasNull: false sum: 5 - Column 9: count: 2 hasNull: false min: bye max: hi sum: 5 - Column 10: count: 2 hasNull: false - Column 11: count: 2 hasNull: false - Column 12: count: 4 hasNull: false - Column 13: count: 4 hasNull: false min: 1 max: 2 sum: 6 - Column 14: count: 4 hasNull: false min: bye max: sigh sum: 14 - Column 15: count: 2 hasNull: false - Column 16: count: 5 hasNull: false - Column 17: count: 5 hasNull: false min: -100000 max: 100000000 sum: 99901241 - Column 18: count: 5 hasNull: false min: bad max: in sum: 15 - Column 19: count: 2 hasNull: false - Column 20: count: 2 hasNull: false min: chani max: mauddib sum: 12 - Column 21: count: 2 hasNull: false - Column 22: count: 2 hasNull: false min: 1 max: 5 sum: 6 - Column 23: count: 2 hasNull: false min: chani max: mauddib sum: 12 - -File Statistics: - Column 0: count: 2 hasNull: false - Column 1: count: 2 hasNull: false true: 1 - Column 2: count: 2 hasNull: false min: 1 max: 100 sum: 101 - Column 3: count: 2 hasNull: false min: 1024 max: 2048 sum: 3072 - Column 4: count: 2 hasNull: false min: 65536 max: 65536 sum: 131072 - Column 5: count: 2 hasNull: false min: 9223372036854775807 max: 9223372036854775807 - Column 6: count: 2 hasNull: false min: 1.0 max: 2.0 sum: 3.0 - Column 7: count: 2 hasNull: false min: -15.0 max: -5.0 sum: -20.0 - Column 8: count: 2 hasNull: false sum: 5 - Column 9: count: 2 hasNull: false min: bye max: hi sum: 5 - Column 10: count: 2 hasNull: false - Column 11: count: 2 hasNull: false - Column 12: count: 4 hasNull: false - Column 13: count: 4 hasNull: false min: 1 max: 2 sum: 6 - Column 14: count: 4 hasNull: false min: bye max: sigh sum: 14 - Column 15: count: 2 hasNull: false - Column 16: count: 5 hasNull: false - Column 17: count: 5 hasNull: false min: -100000 max: 100000000 sum: 99901241 - Column 18: count: 5 hasNull: false min: bad max: in sum: 15 - Column 19: count: 2 hasNull: false - Column 20: count: 2 hasNull: false min: chani max: mauddib sum: 12 - Column 21: count: 2 hasNull: false - Column 22: count: 2 hasNull: false min: 1 max: 5 sum: 6 - Column 23: count: 2 hasNull: false min: chani max: mauddib sum: 12 - -Stripes: - Stripe: offset: 3 data: 243 rows: 2 tail: 199 index: 570 - Stream: column 0 section ROW_INDEX start: 3 length 11 - Stream: column 1 section ROW_INDEX start: 14 length 22 - Stream: column 2 section ROW_INDEX start: 36 length 26 - Stream: column 3 section ROW_INDEX start: 62 length 27 - Stream: column 4 section ROW_INDEX start: 89 length 30 - Stream: column 5 section ROW_INDEX start: 119 length 28 - Stream: column 6 section ROW_INDEX start: 147 length 34 - Stream: column 7 section ROW_INDEX start: 181 length 34 - Stream: column 8 section ROW_INDEX start: 215 length 21 - Stream: column 9 section ROW_INDEX start: 236 length 30 - Stream: column 10 section ROW_INDEX start: 266 length 11 - Stream: column 11 section ROW_INDEX start: 277 length 16 - Stream: column 12 section ROW_INDEX start: 293 length 11 - Stream: column 13 section ROW_INDEX start: 304 length 24 - Stream: column 14 section ROW_INDEX start: 328 length 31 - Stream: column 15 section ROW_INDEX start: 359 length 16 - Stream: column 16 section ROW_INDEX start: 375 length 11 - Stream: column 17 section ROW_INDEX start: 386 length 32 - Stream: column 18 section ROW_INDEX start: 418 length 30 - Stream: column 19 section ROW_INDEX start: 448 length 16 - Stream: column 20 section ROW_INDEX start: 464 length 37 - Stream: column 21 section ROW_INDEX start: 501 length 11 - Stream: column 22 section ROW_INDEX start: 512 length 24 - Stream: column 23 section ROW_INDEX start: 536 length 37 - Stream: column 1 section DATA start: 573 length 5 - Stream: column 2 section DATA start: 578 length 6 - Stream: column 3 section DATA start: 584 length 9 - Stream: column 4 section DATA start: 593 length 11 - Stream: column 5 section DATA start: 604 length 12 - Stream: column 6 section DATA start: 616 length 11 - Stream: column 7 section DATA start: 627 length 15 - Stream: column 8 section DATA start: 642 length 8 - Stream: column 8 section LENGTH start: 650 length 6 - Stream: column 9 section DATA start: 656 length 8 - Stream: column 9 section LENGTH start: 664 length 6 - Stream: column 11 section LENGTH start: 670 length 6 - Stream: column 13 section DATA start: 676 length 7 - Stream: column 14 section DATA start: 683 length 6 - Stream: column 14 section LENGTH start: 689 length 6 - Stream: column 14 section DICTIONARY_DATA start: 695 length 10 - Stream: column 15 section LENGTH start: 705 length 6 - Stream: column 17 section DATA start: 711 length 25 - Stream: column 18 section DATA start: 736 length 18 - Stream: column 18 section LENGTH start: 754 length 8 - Stream: column 19 section LENGTH start: 762 length 6 - Stream: column 20 section DATA start: 768 length 15 - Stream: column 20 section LENGTH start: 783 length 6 - Stream: column 22 section DATA start: 789 length 6 - Stream: column 23 section DATA start: 795 length 15 - Stream: column 23 section LENGTH start: 810 length 6 - Encoding column 0: DIRECT - Encoding column 1: DIRECT - Encoding column 2: DIRECT - Encoding column 3: DIRECT_V2 - Encoding column 4: DIRECT_V2 - Encoding column 5: DIRECT_V2 - Encoding column 6: DIRECT - Encoding column 7: DIRECT - Encoding column 8: DIRECT_V2 - Encoding column 9: DIRECT_V2 - Encoding column 10: DIRECT - Encoding column 11: DIRECT_V2 - Encoding column 12: DIRECT - Encoding column 13: DIRECT_V2 - Encoding column 14: DICTIONARY_V2[2] - Encoding column 15: DIRECT_V2 - Encoding column 16: DIRECT - Encoding column 17: DIRECT_V2 - Encoding column 18: DIRECT_V2 - Encoding column 19: DIRECT_V2 - Encoding column 20: DIRECT_V2 - Encoding column 21: DIRECT - Encoding column 22: DIRECT_V2 - Encoding column 23: DIRECT_V2 - -File length: 1711 bytes -Padding length: 0 bytes -Padding ratio: 0% -______________________________________________________________________ -</code></pre> - -<h2 id="java-data">Java Data</h2> - -<p>The data command prints the data in an ORC file as a JSON document. Each -record is printed as a JSON object on a line. Each record is annotated with -the fieldnames and a JSON representation that depends on the fieldâs type.</p> - -<h2 id="java-scan">Java Scan</h2> - -<p>The scan command reads the contents of the file without printing anything. It -is primarily intendend for benchmarking the Java reader without including the -cost of printing the data out.</p> - -<h2 id="java-convert">Java Convert</h2> - -<p>The convert command reads several JSON files and converts them into a -single ORC file.</p> - -<dl> - <dt>-o <filename></filename></dt> - <dd>Sets the output ORC filename, which defaults to output.orc</dd> - <dt>-s <schema></schema></dt> - <dd>Sets the schema for the ORC file. By default, the schema is automatically discovered.</dd> - <dt>-h</dt> - <dd>Print help</dd> -</dl> - -<p>The automatic JSON schema discovery is equivalent to the json-schema tool -below.</p> - -<h2 id="java-json-schema">Java JSON Schema</h2> - -<p>The JSON Schema discovery tool processes a set of JSON documents and -produces a schema that encompasses all of the records in all of the -documents. It works by computing the enclosing type and promoting it -to include all of the observed values.</p> - -<dl> - <dt>-f</dt> - <dd>Print the schema as a list of flat types for each subfield</dd> - <dt>-t</dt> - <dd>Print the schema as a Hive table declaration</dd> - <dt>-h</dt> - <dd>Print help</dd> -</dl> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <div class="section-nav"> - <div class="left align-right"> - - - - <a href="/docs/cpp-tools.html" class="prev">Back</a> - - </div> - <div class="right align-left"> - - - - <a href="/docs/spec-intro.html" class="next">Next</a> - - </div> - </div> - <div class="clear"></div> - - - </article> - </div> - - <div class="unit one-fifth hide-on-mobiles"> - <aside> - - <h4>Overview</h4> - - -<ul> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/index.html">Background</a></li> - - - - - - - - - - - - - - <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/types.html">Types</a></li> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/indexes.html">Indexes</a></li> - +<p>The subcommands for the tools are:</p> +<ul> + <li>meta - print the metadata of an ORC file</li> + <li>data - print the data of an ORC file</li> + <li>scan (since ORC 1.3) - scan the data for benchmarking</li> + <li>convert (since ORC 1.4) - convert JSON files to ORC</li> + <li>json-schema (since ORC 1.4) - determine the schema of JSON documents</li> +</ul> - +<p>The command line looks like:</p> - - - +<pre><code class="language-shell">% java -jar orc-tools-X.Y.Z-uber.jar <sub-command> <args> +</code></pre> - - - <li class=""><a href="/docs/acid.html">ACID support</a></li> - +<h2 id="java-meta">Java Meta</h2> +<p>The meta command prints the metadata about the given ORC file and is +equivalent to the Hive ORC File Dump command.</p> -</ul> +<dl> + <dt>-j</dt> + <dd>format the output in JSON</dd> + <dt>-p</dt> + <dd>pretty print the output</dd> + <dt>-t</dt> + <dd>print the timezone of the writer</dd> + <dt>ârowindex</dt> + <dd>print the row indexes for the comma separated list of column ids</dd> + <dt>ârecover</dt> + <dd>skip over corrupted values in the ORC file</dd> + <dt>âskip-dump</dt> + <dd>skip dumping the metadata</dd> + <dt>âbackup-path</dt> + <dd>when used with ârecover specifies the path where the recovered file is written</dd> +</dl> - - <h4>Installing</h4> - +<p>An example of the output is given below:</p> -<ul> +<pre><code class="language-shell">% java -jar orc-tools-X.Y.Z-uber.jar meta examples/TestOrcFile.test1.orc +Processing data file examples/TestOrcFile.test1.orc [length: 1711] +Structure for examples/TestOrcFile.test1.orc +File Version: 0.12 with HIVE_8732 +Rows: 2 +Compression: ZLIB +Compression size: 10000 +Type: struct<boolean1:boolean,byte1:tinyint,short1:smallint,int1:int, +long1:bigint,float1:float,double1:double,bytes1:binary,string1:string, +middle:struct<list:array<struct<int1:int,string1:string>>>,list:array< +struct<int1:int,string1:string>>,map:map<string,struct<int1:int,string1: +string>>> - +Stripe Statistics: + Stripe 1: + Column 0: count: 2 hasNull: false + Column 1: count: 2 hasNull: false true: 1 + Column 2: count: 2 hasNull: false min: 1 max: 100 sum: 101 + Column 3: count: 2 hasNull: false min: 1024 max: 2048 sum: 3072 + Column 4: count: 2 hasNull: false min: 65536 max: 65536 sum: 131072 + Column 5: count: 2 hasNull: false min: 9223372036854775807 max: 9223372036854775807 + Column 6: count: 2 hasNull: false min: 1.0 max: 2.0 sum: 3.0 + Column 7: count: 2 hasNull: false min: -15.0 max: -5.0 sum: -20.0 + Column 8: count: 2 hasNull: false sum: 5 + Column 9: count: 2 hasNull: false min: bye max: hi sum: 5 + Column 10: count: 2 hasNull: false + Column 11: count: 2 hasNull: false + Column 12: count: 4 hasNull: false + Column 13: count: 4 hasNull: false min: 1 max: 2 sum: 6 + Column 14: count: 4 hasNull: false min: bye max: sigh sum: 14 + Column 15: count: 2 hasNull: false + Column 16: count: 5 hasNull: false + Column 17: count: 5 hasNull: false min: -100000 max: 100000000 sum: 99901241 + Column 18: count: 5 hasNull: false min: bad max: in sum: 15 + Column 19: count: 2 hasNull: false + Column 20: count: 2 hasNull: false min: chani max: mauddib sum: 12 + Column 21: count: 2 hasNull: false + Column 22: count: 2 hasNull: false min: 1 max: 5 sum: 6 + Column 23: count: 2 hasNull: false min: chani max: mauddib sum: 12 - - - +File Statistics: + Column 0: count: 2 hasNull: false + Column 1: count: 2 hasNull: false true: 1 + Column 2: count: 2 hasNull: false min: 1 max: 100 sum: 101 + Column 3: count: 2 hasNull: false min: 1024 max: 2048 sum: 3072 + Column 4: count: 2 hasNull: false min: 65536 max: 65536 sum: 131072 + Column 5: count: 2 hasNull: false min: 9223372036854775807 max: 9223372036854775807 + Column 6: count: 2 hasNull: false min: 1.0 max: 2.0 sum: 3.0 + Column 7: count: 2 hasNull: false min: -15.0 max: -5.0 sum: -20.0 + Column 8: count: 2 hasNull: false sum: 5 + Column 9: count: 2 hasNull: false min: bye max: hi sum: 5 + Column 10: count: 2 hasNull: false + Column 11: count: 2 hasNull: false + Column 12: count: 4 hasNull: false + Column 13: count: 4 hasNull: false min: 1 max: 2 sum: 6 + Column 14: count: 4 hasNull: false min: bye max: sigh sum: 14 + Column 15: count: 2 hasNull: false + Column 16: count: 5 hasNull: false + Column 17: count: 5 hasNull: false min: -100000 max: 100000000 sum: 99901241 + Column 18: count: 5 hasNull: false min: bad max: in sum: 15 + Column 19: count: 2 hasNull: false + Column 20: count: 2 hasNull: false min: chani max: mauddib sum: 12 + Column 21: count: 2 hasNull: false + Column 22: count: 2 hasNull: false min: 1 max: 5 sum: 6 + Column 23: count: 2 hasNull: false min: chani max: mauddib sum: 12 - - - - - - - <li class=""><a href="/docs/building.html">Building ORC</a></li> - +Stripes: + Stripe: offset: 3 data: 243 rows: 2 tail: 199 index: 570 + Stream: column 0 section ROW_INDEX start: 3 length 11 + Stream: column 1 section ROW_INDEX start: 14 length 22 + Stream: column 2 section ROW_INDEX start: 36 length 26 + Stream: column 3 section ROW_INDEX start: 62 length 27 + Stream: column 4 section ROW_INDEX start: 89 length 30 + Stream: column 5 section ROW_INDEX start: 119 length 28 + Stream: column 6 section ROW_INDEX start: 147 length 34 + Stream: column 7 section ROW_INDEX start: 181 length 34 + Stream: column 8 section ROW_INDEX start: 215 length 21 + Stream: column 9 section ROW_INDEX start: 236 length 30 + Stream: column 10 section ROW_INDEX start: 266 length 11 + Stream: column 11 section ROW_INDEX start: 277 length 16 + Stream: column 12 section ROW_INDEX start: 293 length 11 + Stream: column 13 section ROW_INDEX start: 304 length 24 + Stream: column 14 section ROW_INDEX start: 328 length 31 + Stream: column 15 section ROW_INDEX start: 359 length 16 + Stream: column 16 section ROW_INDEX start: 375 length 11 + Stream: column 17 section ROW_INDEX start: 386 length 32 + Stream: column 18 section ROW_INDEX start: 418 length 30 + Stream: column 19 section ROW_INDEX start: 448 length 16 + Stream: column 20 section ROW_INDEX start: 464 length 37 + Stream: column 21 section ROW_INDEX start: 501 length 11 + Stream: column 22 section ROW_INDEX start: 512 length 24 + Stream: column 23 section ROW_INDEX start: 536 length 37 + Stream: column 1 section DATA start: 573 length 5 + Stream: column 2 section DATA start: 578 length 6 + Stream: column 3 section DATA start: 584 length 9 + Stream: column 4 section DATA start: 593 length 11 + Stream: column 5 section DATA start: 604 length 12 + Stream: column 6 section DATA start: 616 length 11 + Stream: column 7 section DATA start: 627 length 15 + Stream: column 8 section DATA start: 642 length 8 + Stream: column 8 section LENGTH start: 650 length 6 + Stream: column 9 section DATA start: 656 length 8 + Stream: column 9 section LENGTH start: 664 length 6 + Stream: column 11 section LENGTH start: 670 length 6 + Stream: column 13 section DATA start: 676 length 7 + Stream: column 14 section DATA start: 683 length 6 + Stream: column 14 section LENGTH start: 689 length 6 + Stream: column 14 section DICTIONARY_DATA start: 695 length 10 + Stream: column 15 section LENGTH start: 705 length 6 + Stream: column 17 section DATA start: 711 length 25 + Stream: column 18 section DATA start: 736 length 18 + Stream: column 18 section LENGTH start: 754 length 8 + Stream: column 19 section LENGTH start: 762 length 6 + Stream: column 20 section DATA start: 768 length 15 + Stream: column 20 section LENGTH start: 783 length 6 + Stream: column 22 section DATA start: 789 length 6 + Stream: column 23 section DATA start: 795 length 15 + Stream: column 23 section LENGTH start: 810 length 6 + Encoding column 0: DIRECT + Encoding column 1: DIRECT + Encoding column 2: DIRECT + Encoding column 3: DIRECT_V2 + Encoding column 4: DIRECT_V2 + Encoding column 5: DIRECT_V2 + Encoding column 6: DIRECT + Encoding column 7: DIRECT + Encoding column 8: DIRECT_V2 + Encoding column 9: DIRECT_V2 + Encoding column 10: DIRECT + Encoding column 11: DIRECT_V2 + Encoding column 12: DIRECT + Encoding column 13: DIRECT_V2 + Encoding column 14: DICTIONARY_V2[2] + Encoding column 15: DIRECT_V2 + Encoding column 16: DIRECT + Encoding column 17: DIRECT_V2 + Encoding column 18: DIRECT_V2 + Encoding column 19: DIRECT_V2 + Encoding column 20: DIRECT_V2 + Encoding column 21: DIRECT + Encoding column 22: DIRECT_V2 + Encoding column 23: DIRECT_V2 +File length: 1711 bytes +Padding length: 0 bytes +Padding ratio: 0% +______________________________________________________________________ +</code></pre> - +<h2 id="java-data">Java Data</h2> - - - +<p>The data command prints the data in an ORC file as a JSON document. Each +record is printed as a JSON object on a line. Each record is annotated with +the fieldnames and a JSON representation that depends on the fieldâs type.</p> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/releases.html">Releases</a></li> - +<h2 id="java-scan">Java Scan</h2> + +<p>The scan command reads the contents of the file without printing anything. It +is primarily intendend for benchmarking the Java reader without including the +cost of printing the data out.</p> + +<h2 id="java-convert">Java Convert</h2> +<p>The convert command reads several JSON files and converts them into a +single ORC file.</p> + +<dl> + <dt>-o <filename></filename></dt> + <dd>Sets the output ORC filename, which defaults to output.orc</dd> + <dt>-s <schema></schema></dt> + <dd>Sets the schema for the ORC file. By default, the schema is automatically discovered.</dd> + <dt>-h</dt> + <dd>Print help</dd> +</dl> + +<p>The automatic JSON schema discovery is equivalent to the json-schema tool +below.</p> + +<h2 id="java-json-schema">Java JSON Schema</h2> + +<p>The JSON Schema discovery tool processes a set of JSON documents and +produces a schema that encompasses all of the records in all of the +documents. It works by computing the enclosing type and promoting it +to include all of the observed values.</p> + +<dl> + <dt>-f</dt> + <dd>Print the schema as a list of flat types for each subfield</dd> + <dt>-t</dt> + <dd>Print the schema as a Hive table declaration</dd> + <dt>-h</dt> + <dd>Print help</dd> +</dl> + + -</ul> - - <h4>Using in Hive</h4> - -<ul> - - - - + - - + - - + - - + - - + + - - <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li> - + + + - - - + - - + - - + - - + - + <div class="section-nav"> + <div class="left align-right"> + + + + <a href="/docs/cpp-tools.html" class="prev">Back</a> + + </div> + <div class="right align-left"> + + <span class="next disabled">Next</span> + + </div> + </div> + <div class="clear"></div> - <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li> - + </article> + </div> -</ul> - + <div class="unit one-fifth hide-on-mobiles"> + <aside> - <h4>Using in MapReduce</h4> + <h4>Overview</h4> <ul> @@ -1879,19 +1014,21 @@ to include all of the observed values.</p> + <li class=""><a href="/docs/index.html">Background</a></li> + + + - - - + - + - <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li> + <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li> @@ -1931,20 +1068,10 @@ to include all of the observed values.</p> - - - <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li> + <li class=""><a href="/docs/types.html">Types</a></li> -</ul> - - - <h4>Using ORC Core</h4> - - -<ul> - @@ -1963,34 +1090,34 @@ to include all of the observed values.</p> - <li class=""><a href="/docs/core-java.html">Using Core Java</a></li> - - - - - - - + <li class=""><a href="/docs/indexes.html">Indexes</a></li> + + + + + + + - <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li> + <li class=""><a href="/docs/acid.html">ACID support</a></li> </ul> - <h4>Tools</h4> + <h4>Installing</h4> <ul> @@ -2007,15 +1134,7 @@ to include all of the observed values.</p> - - - - - - - - - <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li> + <li class=""><a href="/docs/building.html">Building ORC</a></li> @@ -2053,14 +1172,14 @@ to include all of the observed values.</p> - <li class="current"><a href="/docs/java-tools.html">Java Tools</a></li> + <li class=""><a href="/docs/releases.html">Releases</a></li> </ul> - <h4>Format Specification</h4> + <h4>Using in Hive</h4> <ul> @@ -2087,31 +1206,7 @@ to include all of the observed values.</p> - - - - - - - - - - - - - - - - - - - - - - - - - <li class=""><a href="/docs/spec-intro.html">Introduction</a></li> + <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li> @@ -2135,31 +1230,17 @@ to include all of the observed values.</p> - - - - - <li class=""><a href="/docs/file-tail.html">File Tail</a></li> + <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li> - - - - - +</ul> - - - - - - + <h4>Using in MapReduce</h4> - <li class=""><a href="/docs/compression.html">Compression</a></li> - +<ul> @@ -2191,19 +1272,7 @@ to include all of the observed values.</p> - - - - - - - - - - - - - <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li> + <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li> @@ -2239,13 +1308,25 @@ to include all of the observed values.</p> - + <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li> + + + +</ul> + - + <h4>Using ORC Core</h4> + +<ul> + + + + + @@ -2255,7 +1336,7 @@ to include all of the observed values.</p> - <li class=""><a href="/docs/stripes.html">Stripes</a></li> + <li class=""><a href="/docs/core-java.html">Using Core Java</a></li> @@ -2273,17 +1354,17 @@ to include all of the observed values.</p> - - - - - + <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li> + + + +</ul> + - + <h4>Tools</h4> - <li class=""><a href="/docs/encodings.html">Column Encodings</a></li> - +<ul> @@ -2303,11 +1384,17 @@ to include all of the observed values.</p> + <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li> + + + - + + + @@ -2329,7 +1416,7 @@ to include all of the observed values.</p> - <li class=""><a href="/docs/spec-index.html">Indexes</a></li> + <li class="current"><a href="/docs/java-tools.html">Java Tools</a></li>
