This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 1569b20 Deployed 3a35c0764 with MkDocs version: 1.0.4
1569b20 is described below
commit 1569b203e1340c6a91844b0d5aa9e98f89a98155
Author: Ryan Blue <[email protected]>
AuthorDate: Sat Apr 11 12:26:03 2020 -0800
Deployed 3a35c0764 with MkDocs version: 1.0.4
---
404.html | 20 ++-
api-quickstart/index.html | 38 +++--
api/index.html | 25 ++-
community/index.html | 20 ++-
configuration/index.html | 77 ++++++++--
custom-catalog/index.html | 20 ++-
disclaimer/index.html | 20 ++-
evolution/index.html | 23 ++-
getting-started/index.html | 92 +++++++++--
how-to-release/index.html | 22 ++-
img/partition-spec-evolution.png | Bin 0 -> 224020 bytes
index.html | 22 ++-
{api-quickstart => java-api-quickstart}/index.html | 101 +++++++------
partitioning/index.html | 24 +--
performance/index.html | 22 ++-
presto/index.html | 22 ++-
python-api-intro/index.html | 24 +--
python-feature-support/index.html | 20 ++-
python-quickstart/index.html | 20 ++-
releases/index.html | 20 ++-
reliability/index.html | 20 ++-
schemas/index.html | 24 +--
sitemap.xml | 49 +++---
sitemap.xml.gz | Bin 221 -> 220 bytes
snapshots/index.html | 20 ++-
spark/index.html | 20 ++-
spec/index.html | 168 ++++++++++++++++++---
terms/index.html | 20 ++-
why-iceberg/index.html | 20 ++-
29 files changed, 687 insertions(+), 286 deletions(-)
diff --git a/404.html b/404.html
index c7cc71e..6a98545 100644
--- a/404.html
+++ b/404.html
@@ -116,19 +116,25 @@
<li >
- <a href="/schemas/">Schemas</a>
+ <a href="/getting-started/">Getting Started</a>
</li>
<li >
- <a href="/partitioning/">Partitioning</a>
+ <a href="/configuration/">Configuration</a>
</li>
<li >
- <a href="/configuration/">Configuration</a>
+ <a href="/schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="/partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="/spark#time-travel">Time travel</a>
+ <a href="/spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="/api-quickstart/">Quickstart</a>
+ <a href="/api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="/api-quickstart/">Quickstart</a>
+ <a href="/java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="/python-api-intro/">API intro</a>
+ <a href="/python-api-intro/">API Intro</a>
</li>
diff --git a/api-quickstart/index.html b/api-quickstart/index.html
index f37db68..d1a9c1a 100644
--- a/api-quickstart/index.html
+++ b/api-quickstart/index.html
@@ -11,7 +11,7 @@
<link rel="../img/favicon.ico">
- <title>Quickstart - Apache Iceberg (incubating)</title>
+ <title>Spark Quickstart - Apache Iceberg (incubating)</title>
<link rel="stylesheet"
href="//use.fontawesome.com/releases/v5.5.0/css/all.css"
integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU"
crossorigin="anonymous">
@@ -110,25 +110,31 @@
- <li class="dropdown">
+ <li class="dropdown active">
<a href="#" class="dropdown-toggle"
data-toggle="dropdown">User docs <b class="caret"></b></a>
<ul class="dropdown-menu">
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
-<li >
- <a href="./">Quickstart</a>
+<li class="active">
+ <a href="./">Spark Quickstart</a>
</li>
@@ -179,7 +185,7 @@
- <li class="dropdown active">
+ <li class="dropdown">
<a href="#" class="dropdown-toggle"
data-toggle="dropdown">Java <b class="caret"></b></a>
<ul class="dropdown-menu">
@@ -190,8 +196,8 @@
-<li class="active">
- <a href="./">Quickstart</a>
+<li >
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -281,12 +287,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../presto/">
+ <a rel="prev" href="../evolution/">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../api/">
+ <a rel="next" href="../spark/">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -300,7 +306,7 @@
<div class="col-md-3"><div class="bs-sidebar hidden-print affix well"
role="complementary">
<ul class="nav bs-sidenav">
- <li class="first-level active"><a href="#api-quickstart">API
Quickstart</a></li>
+ <li class="first-level active"><a href="#spark-api-quickstart">Spark
API Quickstart</a></li>
<li class="second-level"><a href="#create-a-table">Create a
table</a></li>
<li class="third-level"><a href="#using-a-hive-catalog">Using
a Hive catalog</a></li>
@@ -335,7 +341,7 @@
- limitations under the License.
-->
-<h1 id="api-quickstart">API Quickstart<a class="headerlink"
href="#api-quickstart" title="Permanent link">¶</a></h1>
+<h1 id="spark-api-quickstart">Spark API Quickstart<a class="headerlink"
href="#spark-api-quickstart" title="Permanent link">¶</a></h1>
<h2 id="create-a-table">Create a table<a class="headerlink"
href="#create-a-table" title="Permanent link">¶</a></h2>
<p>Tables are created using either a <a
href="/javadoc/master/index.html?org/apache/iceberg/catalog/Catalog.html"><code>Catalog</code></a>
or an implementation of the <a
href="/javadoc/master/index.html?org/apache/iceberg/Tables.html"><code>Tables</code></a>
interface.</p>
<h3 id="using-a-hive-catalog">Using a Hive catalog<a class="headerlink"
href="#using-a-hive-catalog" title="Permanent link">¶</a></h3>
diff --git a/api/index.html b/api/index.html
index f1881ab..30d4d6c 100644
--- a/api/index.html
+++ b/api/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -281,7 +287,7 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../api-quickstart/">
+ <a rel="prev" href="../java-api-quickstart/">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
@@ -444,6 +450,9 @@ ListType list = ListType.ofRequired(1, IntegerType.get());
<li><code>lessThanOrEqual</code></li>
<li><code>greaterThan</code></li>
<li><code>greaterThanOrEqual</code></li>
+<li><code>in</code></li>
+<li><code>notIn</code></li>
+<li><code>startsWith</code></li>
</ul>
<p>Supported expression operations are:</p>
<ul>
diff --git a/community/index.html b/community/index.html
index 0cd37b8..f1f08be 100644
--- a/community/index.html
+++ b/community/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/configuration/index.html b/configuration/index.html
index cc70c50..27535cb 100644
--- a/configuration/index.html
+++ b/configuration/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
+</li>
+
+
+
+<li class="active">
+ <a href="./">Configuration</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../schemas/">Schemas</a>
</li>
-<li class="active">
- <a href="./">Configuration</a>
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -281,12 +287,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../partitioning/">
+ <a rel="prev" href="../getting-started/">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../performance/">
+ <a rel="next" href="../schemas/">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -306,6 +312,9 @@
<li class="third-level"><a href="#read-properties">Read
properties</a></li>
<li class="third-level"><a href="#write-properties">Write
properties</a></li>
<li class="third-level"><a
href="#table-behavior-properties">Table behavior properties</a></li>
+ <li class="third-level"><a
href="#compatibility-flags">Compatibility flags</a></li>
+ <li class="second-level"><a href="#hadoop-options">Hadoop
options</a></li>
+
<li class="second-level"><a href="#spark-options">Spark
options</a></li>
<li class="third-level"><a href="#read-options">Read
options</a></li>
@@ -347,7 +356,12 @@
<tr>
<td>read.split.target-size</td>
<td>134217728 (128 MB)</td>
-<td>Target size when combining input splits</td>
+<td>Target size when combining data input splits</td>
+</tr>
+<tr>
+<td>read.split.metadata-target-size</td>
+<td>33554432 (32 MB)</td>
+<td>Target size when combining metadata input splits</td>
</tr>
<tr>
<td>read.split.planning-lookback</td>
@@ -490,6 +504,45 @@
</tr>
</tbody>
</table>
+<h3 id="compatibility-flags">Compatibility flags<a class="headerlink"
href="#compatibility-flags" title="Permanent link">¶</a></h3>
+<table>
+<thead>
+<tr>
+<th>Property</th>
+<th>Default</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>compatibility.snapshot-id-inheritance.enabled</td>
+<td>false</td>
+<td>Enables committing snapshots without explicit snapshot IDs</td>
+</tr>
+</tbody>
+</table>
+<h2 id="hadoop-options">Hadoop options<a class="headerlink"
href="#hadoop-options" title="Permanent link">¶</a></h2>
+<table>
+<thead>
+<tr>
+<th>Property</th>
+<th>Default</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>iceberg.hive.client-pool-size</td>
+<td>5</td>
+<td>The size of the Hive client pool when tracking tables in HMS</td>
+</tr>
+<tr>
+<td>iceberg.hive.lock-timeout-ms</td>
+<td>180000 (3 min)</td>
+<td>Maximum time in milliseconds to acquire a lock</td>
+</tr>
+</tbody>
+</table>
<h2 id="spark-options">Spark options<a class="headerlink"
href="#spark-options" title="Permanent link">¶</a></h2>
<h3 id="read-options">Read options<a class="headerlink" href="#read-options"
title="Permanent link">¶</a></h3>
<p>Spark read options are passed when configuring the DataFrameReader, like
this:</p>
@@ -522,7 +575,7 @@ spark.read
<tr>
<td>split-size</td>
<td>As per table property</td>
-<td>Overrides this table’s read.split.target-size</td>
+<td>Overrides this table’s read.split.target-size and
read.split.metadata-target-size</td>
</tr>
<tr>
<td>lookback</td>
diff --git a/custom-catalog/index.html b/custom-catalog/index.html
index d506492..5c8e931 100644
--- a/custom-catalog/index.html
+++ b/custom-catalog/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/disclaimer/index.html b/disclaimer/index.html
index 3052906..7c7b59f 100644
--- a/disclaimer/index.html
+++ b/disclaimer/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/evolution/index.html b/evolution/index.html
index 222373f..2dbe510 100644
--- a/evolution/index.html
+++ b/evolution/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -356,6 +362,9 @@
</ul>
<h2 id="partition-evolution">Partition evolution<a class="headerlink"
href="#partition-evolution" title="Permanent link">¶</a></h2>
<p>Iceberg table partitioning can be updated in an existing table because
queries do not reference partition values directly.</p>
+<p>When you evolve a partition spec, the old data written with an earlier spec
remains unchanged. New data is written using the new spec in a new layout.
Metadata for each of the partition versions is kept separately. Because of
this, when you start writing queries, you get split planning. This is where
each partition layout plans files separately using the filter it derives for
that specific partition layout. Here’s a visual representation of a
contrived example: </p>
+<p><img alt="Partition evolution diagram"
src="../img/partition-spec-evolution.png" />
+<em>The data for 2008 is partitioned by month. Starting from 2009 the table is
updated so that the data is instead partitioned by day. Both partitioning
layouts are able to coexist in the same table.</em></p>
<p>Iceberg uses <a href="../partitioning">hidden partitioning</a>, so you
don’t <em>need</em> to write queries for a specific partition layout to
be fast. Instead, you can write queries that select the data you need, and
Iceberg automatically prunes out files that don’t contain matching
data.</p>
<p>Partition evolution is a metadata operation and does not eagerly rewrite
files.</p></div>
diff --git a/getting-started/index.html b/getting-started/index.html
index 9b364be..f4d6096 100644
--- a/getting-started/index.html
+++ b/getting-started/index.html
@@ -11,7 +11,7 @@
<link rel="../img/favicon.ico">
- <title>Getting started - Apache Iceberg (incubating)</title>
+ <title>Getting Started - Apache Iceberg (incubating)</title>
<link rel="stylesheet"
href="//use.fontawesome.com/releases/v5.5.0/css/all.css"
integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU"
crossorigin="anonymous">
@@ -110,25 +110,31 @@
- <li class="dropdown">
+ <li class="dropdown active">
<a href="#" class="dropdown-toggle"
data-toggle="dropdown">User docs <b class="caret"></b></a>
<ul class="dropdown-menu">
+<li class="active">
+ <a href="./">Getting Started</a>
+</li>
+
+
+
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../schemas/">Schemas</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -280,6 +286,16 @@
</ul>
<ul class="nav navbar-nav navbar-right">
+ <li >
+ <a rel="prev" href="../how-to-release/">
+ <i class="fas fa-arrow-left"></i> Previous
+ </a>
+ </li>
+ <li >
+ <a rel="next" href="../configuration/">
+ Next <i class="fas fa-arrow-right"></i>
+ </a>
+ </li>
</ul>
</div>
</div>
@@ -291,6 +307,12 @@
<div class="col-md-3"><div class="bs-sidebar hidden-print affix well"
role="complementary">
<ul class="nav bs-sidenav">
<li class="first-level active"><a href="#getting-started">Getting
Started</a></li>
+ <li class="second-level"><a href="#including-iceberg">Including
Iceberg</a></li>
+
+ <li class="third-level"><a href="#downloads">Downloads</a></li>
+ <li class="third-level"><a href="#gradle">Gradle</a></li>
+ <li class="third-level"><a href="#maven">Maven</a></li>
+ <li class="third-level"><a href="#using-the-api">Using the
API</a></li>
</ul>
</div></div>
<div class="col-md-9" role="main">
@@ -312,7 +334,57 @@
- limitations under the License.
-->
-<h2 id="getting-started">Getting Started<a class="headerlink"
href="#getting-started" title="Permanent link">¶</a></h2></div>
+<h1 id="getting-started">Getting Started<a class="headerlink"
href="#getting-started" title="Permanent link">¶</a></h1>
+<h2 id="including-iceberg">Including Iceberg<a class="headerlink"
href="#including-iceberg" title="Permanent link">¶</a></h2>
+<h3 id="downloads">Downloads<a class="headerlink" href="#downloads"
title="Permanent link">¶</a></h3>
+<p>The latest version of Iceberg is <a
href="https://github.com/apache/incubator-iceberg/releases/tag/apache-iceberg-0.7.0-incubating";>0.7.0-incubating</a>.</p>
+<ul>
+<li><a
href="https://www.apache.org/dyn/closer.cgi/incubator/iceberg/apache-iceberg-0.7.0-incubating/apache-iceberg-0.7.0-incubating.tar.gz";>0.7.0-incubating
source tar.gz</a> – <a
href="https://dist.apache.org/repos/dist/release/incubator/iceberg/apache-iceberg-0.7.0-incubating/apache-iceberg-0.7.0-incubating.tar.gz.asc";>signature</a>
– <a
href="https://dist.apache.org/repos/dist/release/incubator/iceberg/apache-iceberg-0.7.0-incubating/apache-iceberg-0.7.0-incubating.tar.gz
[...]
+<li><a
href="https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark-runtime/0.7.0-incubating/iceberg-spark-runtime-0.7.0-incubating.jar";>0.7.0-incubating
Spark 2.4 runtime Jar</a></li>
+</ul>
+<p>One way to use Iceberg in Spark 2.4 is to download the runtime Jar and add
it to the jars folder of your Spark install.</p>
+<p>Spark 2.4 is limited to reading and writing existing Iceberg tables. Use
the <a href="../api">Iceberg API</a> to create Iceberg tables.</p>
+<p>The recommended way is to include Iceberg’s latest release using the
<code>--packages</code> option:</p>
+<pre><code class="sh">spark-shell --packages
org.apache.iceberg:iceberg-spark-runtime:0.7.0-incubating
+</code></pre>
+
+<p>You can also build Iceberg locally, and add the jar to Spark’s
classpath. This can be helpful to test unreleased features or while developing
something new:</p>
+<pre><code class="sh">./gradlew assemble
+
+spark-shell --jars spark-runtime/build/libs/iceberg-spark-runtime-93990904.jar
+</code></pre>
+
+<p>Where you have to replace <code>93990904</code> with the git hash that
you’re using.</p>
+<h3 id="gradle">Gradle<a class="headerlink" href="#gradle" title="Permanent
link">¶</a></h3>
+<p>To add a dependency on Iceberg in Gradle, add the following to
<code>build.gradle</code>:</p>
+<pre><code>dependencies {
+ compile 'org.apache.iceberg:iceberg-core:0.7.0-incubating'
+}
+</code></pre>
+
+<h3 id="maven">Maven<a class="headerlink" href="#maven" title="Permanent
link">¶</a></h3>
+<p>If you’d like to try out Iceberg in a Maven project using the Spark
Iceberg API, you can add the <code>iceberg-spark-runtime</code> dependency to
your <code>pom.xml</code> file:</p>
+<pre><code class="xml"> <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-spark-runtime</artifactId>
+ <version>${iceberg.version}</version>
+ </dependency>
+</code></pre>
+
+<p>You’ll also need <code>spark-sql</code> to read tables:</p>
+<pre><code class="xml"> <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.11</artifactId>
+ <version>2.4.4</version>
+ </dependency>
+</code></pre>
+
+<h3 id="using-the-api">Using the API<a class="headerlink"
href="#using-the-api" title="Permanent link">¶</a></h3>
+<p>For examples on how to use the Iceberg API see:</p>
+<ul>
+<li><a href="../api-quickstart/">Spark</a></li>
+<li><a href="../java-api-quickstart/">Java</a></li>
+</ul></div>
</div>
diff --git a/how-to-release/index.html b/how-to-release/index.html
index 8c282f9..431b242 100644
--- a/how-to-release/index.html
+++ b/how-to-release/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -286,7 +292,7 @@
</a>
</li>
<li >
- <a rel="next" href="../schemas/">
+ <a rel="next" href="../getting-started/">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
diff --git a/img/partition-spec-evolution.png b/img/partition-spec-evolution.png
new file mode 100644
index 0000000..0bc595f
Binary files /dev/null and b/img/partition-spec-evolution.png differ
diff --git a/index.html b/index.html
index e8b0e64..b00ea2b 100644
--- a/index.html
+++ b/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="schemas/">Schemas</a>
+ <a href="getting-started/">Getting Started</a>
</li>
<li >
- <a href="partitioning/">Partitioning</a>
+ <a href="configuration/">Configuration</a>
</li>
<li >
- <a href="configuration/">Configuration</a>
+ <a href="schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="spark#time-travel">Time travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="api-quickstart/">Quickstart</a>
+ <a href="api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="api-quickstart/">Quickstart</a>
+ <a href="java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="python-api-intro/">API intro</a>
+ <a href="python-api-intro/">API Intro</a>
</li>
@@ -419,5 +425,5 @@
<!--
MkDocs version : 1.0.4
-Build Date UTC : 2020-01-07 00:43:11
+Build Date UTC : 2020-04-11 20:26:03
-->
diff --git a/api-quickstart/index.html b/java-api-quickstart/index.html
similarity index 86%
copy from api-quickstart/index.html
copy to java-api-quickstart/index.html
index f37db68..0e71523 100644
--- a/api-quickstart/index.html
+++ b/java-api-quickstart/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="./">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -300,7 +306,7 @@
<div class="col-md-3"><div class="bs-sidebar hidden-print affix well"
role="complementary">
<ul class="nav bs-sidenav">
- <li class="first-level active"><a href="#api-quickstart">API
Quickstart</a></li>
+ <li class="first-level active"><a href="#java-api-quickstart">Java API
Quickstart</a></li>
<li class="second-level"><a href="#create-a-table">Create a
table</a></li>
<li class="third-level"><a href="#using-a-hive-catalog">Using
a Hive catalog</a></li>
@@ -335,43 +341,36 @@
- limitations under the License.
-->
-<h1 id="api-quickstart">API Quickstart<a class="headerlink"
href="#api-quickstart" title="Permanent link">¶</a></h1>
+<h1 id="java-api-quickstart">Java API Quickstart<a class="headerlink"
href="#java-api-quickstart" title="Permanent link">¶</a></h1>
<h2 id="create-a-table">Create a table<a class="headerlink"
href="#create-a-table" title="Permanent link">¶</a></h2>
<p>Tables are created using either a <a
href="/javadoc/master/index.html?org/apache/iceberg/catalog/Catalog.html"><code>Catalog</code></a>
or an implementation of the <a
href="/javadoc/master/index.html?org/apache/iceberg/Tables.html"><code>Tables</code></a>
interface.</p>
<h3 id="using-a-hive-catalog">Using a Hive catalog<a class="headerlink"
href="#using-a-hive-catalog" title="Permanent link">¶</a></h3>
<p>The Hive catalog connects to a Hive MetaStore to keep track of Iceberg
tables. This example uses Spark’s Hadoop configuration to get a Hive
catalog:</p>
-<pre><code class="scala">import org.apache.iceberg.hive.HiveCatalog
+<pre><code class="java">import org.apache.iceberg.hive.HiveCatalog;
-val catalog = new HiveCatalog(spark.sessionState.newHadoopConf())
+Catalog catalog = new HiveCatalog(spark.sparkContext().hadoopConfiguration());
</code></pre>
<p>The <code>Catalog</code> interface defines methods for working with tables,
like <code>createTable</code>, <code>loadTable</code>,
<code>renameTable</code>, and <code>dropTable</code>.</p>
<p>To create a table, pass an <code>Identifier</code> and a
<code>Schema</code> along with other initial metadata:</p>
-<pre><code class="scala">val name = TableIdentifier.of("logging",
"logs")
-val table = catalog.createTable(name, schema, spec)
-
-// write into the new logs table with Spark 2.4
-logsDF.write
- .format("iceberg")
- .mode("append")
- .save("logging.logs")
+<pre><code class="java">import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.TableIdentifier;
+
+TableIdentifier name = TableIdentifier.of("logging",
"logs");
+Table table = catalog.createTable(name, schema, spec);
</code></pre>
<p>The logs <a href="#create-a-schema">schema</a> and <a
href="#create-a-partition-spec">partition spec</a> are created below.</p>
<h3 id="using-hadoop-tables">Using Hadoop tables<a class="headerlink"
href="#using-hadoop-tables" title="Permanent link">¶</a></h3>
<p>Iceberg also supports tables that are stored in a directory in HDFS or the
local file system. Directory tables don’t support all catalog operations,
like rename, so they use the <code>Tables</code> interface instead of
<code>Catalog</code>.</p>
<p>To create a table in HDFS, use <code>HadoopTables</code>:</p>
-<pre><code class="scala">import org.apache.iceberg.hadoop.HadoopTables
-
-val tables = new HadoopTables(spark.sessionState.newHadoopConf())
+<pre><code class="java">import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.Table;
-val table = tables.create(schema, spec, "hdfs:/tables/logging/logs")
-
-// write into the new logs table with Spark 2.4
-logsDF.write
- .format("iceberg")
- .mode("append")
- .save("hdfs:/tables/logging/logs")
+Configuration conf = new Configuration():
+HadoopTables tables = new HadoopTables(conf);
+Table table = tables.create(schema, spec, table_location);
</code></pre>
<div class="admonition warning">
@@ -389,47 +388,49 @@ logsDF.write
<h2 id="schemas">Schemas<a class="headerlink" href="#schemas" title="Permanent
link">¶</a></h2>
<h3 id="create-a-schema">Create a schema<a class="headerlink"
href="#create-a-schema" title="Permanent link">¶</a></h3>
<p>This example creates a schema for a <code>logs</code> table:</p>
-<pre><code class="scala">import org.apache.iceberg.Schema
-import org.apache.iceberg.types.Types._
-
-val schema = new Schema(
- NestedField.required(1, "level", StringType.get()),
- NestedField.required(2, "event_time", TimestampType.withZone()),
- NestedField.required(3, "message", StringType.get()),
- NestedField.optional(4, "call_stack", ListType.ofRequired(5,
StringType.get()))
- )
+<pre><code class="java">import org.apache.iceberg.Schema;
+import org.apache.iceberg.types.Types;
+
+Schema schema = new Schema(
+ Types.NestedField.required(1, "level", Types.StringType.get()),
+ Types.NestedField.required(2, "event_time",
Types.TimestampType.withZone()),
+ Types.NestedField.required(3, "message",
Types.StringType.get()),
+ Types.NestedField.optional(4, "call_stack",
Types.ListType.ofRequired(5, Types.StringType.get()))
+ );
</code></pre>
<p>When using the Iceberg API directly, type IDs are required. Conversions
from other schema formats, like Spark, Avro, and Parquet will automatically
assign new IDs.</p>
<p>When a table is created, all IDs in the schema are re-assigned to ensure
uniqueness.</p>
<h3 id="convert-a-schema-from-avro">Convert a schema from Avro<a
class="headerlink" href="#convert-a-schema-from-avro" title="Permanent
link">¶</a></h3>
<p>To create an Iceberg schema from an existing Avro schema, use converters in
<code>AvroSchemaUtil</code>:</p>
-<pre><code class="scala">import org.apache.avro.Schema.Parser
-import org.apache.iceberg.avro.AvroSchemaUtil
-
-val avroSchema = new Parser().parse("""{"type":
"record", ... }""")
+<pre><code class="java">import org.apache.avro.Schema;
+import org.apache.avro.Schema.Parser;
+import org.apache.iceberg.avro.AvroSchemaUtil;
-val icebergSchema = AvroSchemaUtil.toIceberg(avroSchema)
+Schema avroSchema = new Parser().parse("{\"type\":
\"record\" , ... }");
+Schema icebergSchema = AvroSchemaUtil.toIceberg(avroSchema);
</code></pre>
<h3 id="convert-a-schema-from-spark">Convert a schema from Spark<a
class="headerlink" href="#convert-a-schema-from-spark" title="Permanent
link">¶</a></h3>
<p>To create an Iceberg schema from an existing table, use converters in
<code>SparkSchemaUtil</code>:</p>
-<pre><code class="scala">import org.apache.iceberg.spark.SparkSchemaUtil
+<pre><code class="java">import org.apache.iceberg.spark.SparkSchemaUtil;
-val schema = SparkSchemaUtil.convert(spark.table("db.table").schema)
+Schema schema = SparkSchemaUtil.schemaForTable(sparkSession, table_name);
</code></pre>
<h2 id="partitioning">Partitioning<a class="headerlink" href="#partitioning"
title="Permanent link">¶</a></h2>
<h3 id="create-a-partition-spec">Create a partition spec<a class="headerlink"
href="#create-a-partition-spec" title="Permanent link">¶</a></h3>
<p>Partition specs describe how Iceberg should group records into data files.
Partition specs are created for a table’s schema using a builder.</p>
<p>This example creates a partition spec for the <code>logs</code> table that
partitions records by the hour of the log event’s timestamp and by log
level:</p>
-<pre><code class="scala">import org.apache.iceberg.PartitionSpec
+<pre><code class="java">import org.apache.iceberg.PartitionSpec;
+
+PartitionSpec spec = PartitionSpec.builderFor(schema)
+ .hour("event_time")
+ .identity("level")
+ .build();
+</code></pre>
-val spec = PartitionSpec.builderFor(schema)
- .hour("event_time")
- .identity("level")
- .build()
-</code></pre></div>
+<p>For more information on the different partition transforms that Iceberg
offers, visit <a href="../spec#partitioning">this page</a>.</p></div>
</div>
diff --git a/partitioning/index.html b/partitioning/index.html
index 6cacddf..e4a4913 100644
--- a/partitioning/index.html
+++ b/partitioning/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
-<li class="active">
- <a href="./">Partitioning</a>
+<li >
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li class="active">
+ <a href="./">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -286,7 +292,7 @@
</a>
</li>
<li >
- <a rel="next" href="../configuration/">
+ <a rel="next" href="../performance/">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
diff --git a/performance/index.html b/performance/index.html
index 6c9c73a..9d9aa90 100644
--- a/performance/index.html
+++ b/performance/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -281,7 +287,7 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../configuration/">
+ <a rel="prev" href="../partitioning/">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
diff --git a/presto/index.html b/presto/index.html
index 8a5569c..a8f7f6d 100644
--- a/presto/index.html
+++ b/presto/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -286,7 +292,7 @@
</a>
</li>
<li >
- <a rel="next" href="../api-quickstart/">
+ <a rel="next" href="../java-api-quickstart/">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
diff --git a/python-api-intro/index.html b/python-api-intro/index.html
index 91bec90..c23dac5 100644
--- a/python-api-intro/index.html
+++ b/python-api-intro/index.html
@@ -11,7 +11,7 @@
<link rel="../img/favicon.ico">
- <title>API intro - Apache Iceberg (incubating)</title>
+ <title>API Intro - Apache Iceberg (incubating)</title>
<link rel="stylesheet"
href="//use.fontawesome.com/releases/v5.5.0/css/all.css"
integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU"
crossorigin="anonymous">
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li class="active">
- <a href="./">API intro</a>
+ <a href="./">API Intro</a>
</li>
@@ -406,7 +412,7 @@ list_var = ListType.of_required(1, IntegerType.get());
</code></pre>
<h2 id="expressions">Expressions<a class="headerlink" href="#expressions"
title="Permanent link">¶</a></h2>
-<p>Iceberg’s <code>Expressions</code> are used to configure table scans. To
create Expressions<code>, use the factory methods in</code>Expressions`.</p>
+<p>Iceberg’s <code>Expressions</code> are used to configure table scans. To
create <code>Expressions</code>, use the factory methods in
<code>Expressions</code>.</p>
<p>Supported <code>Predicate</code> expressions are:</p>
<ul>
<li><code>is_null</code></li>
diff --git a/python-feature-support/index.html
b/python-feature-support/index.html
index 645beb5..422c4fc 100644
--- a/python-feature-support/index.html
+++ b/python-feature-support/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/python-quickstart/index.html b/python-quickstart/index.html
index b0fe8f2..93b02dd 100644
--- a/python-quickstart/index.html
+++ b/python-quickstart/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/releases/index.html b/releases/index.html
index 0aa4f92..1ca8289 100644
--- a/releases/index.html
+++ b/releases/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/reliability/index.html b/reliability/index.html
index 38984b5..d252484 100644
--- a/reliability/index.html
+++ b/reliability/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/schemas/index.html b/schemas/index.html
index 6fee9a0..1e76d3e 100644
--- a/schemas/index.html
+++ b/schemas/index.html
@@ -115,20 +115,26 @@
<ul class="dropdown-menu">
-<li class="active">
- <a href="./">Schemas</a>
+<li >
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
+</li>
+
+
+
+<li class="active">
+ <a href="./">Schemas</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -281,7 +287,7 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../how-to-release/">
+ <a rel="prev" href="../configuration/">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
diff --git a/sitemap.xml b/sitemap.xml
index d99f123..99eddb3 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,57 +2,62 @@
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9";>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
+ <changefreq>daily</changefreq>
+ </url>
+ <url>
+ <loc>None</loc>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
@@ -62,17 +67,17 @@
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
@@ -82,12 +87,12 @@
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
@@ -97,7 +102,7 @@
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
@@ -107,27 +112,27 @@
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2020-01-06</lastmod>
+ <lastmod>2020-04-11</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 39de9c2..850ef3c 100644
Binary files a/sitemap.xml.gz and b/sitemap.xml.gz differ
diff --git a/snapshots/index.html b/snapshots/index.html
index f32e1d6..617fa92 100644
--- a/snapshots/index.html
+++ b/snapshots/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/spark/index.html b/spark/index.html
index edad352..5844df9 100644
--- a/spark/index.html
+++ b/spark/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/spec/index.html b/spec/index.html
index b439a7c..c658750 100644
--- a/spec/index.html
+++ b/spec/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
@@ -315,6 +321,7 @@
<li class="third-level"><a href="#manifests">Manifests</a></li>
<li class="third-level"><a href="#snapshots">Snapshots</a></li>
<li class="third-level"><a href="#table-metadata">Table
Metadata</a></li>
+ <li class="third-level"><a href="#delete-format">Delete
Format</a></li>
<li class="second-level"><a
href="#appendix-a-format-specific-requirements">Appendix A: Format-specific
Requirements</a></li>
<li class="third-level"><a href="#avro">Avro</a></li>
@@ -329,6 +336,9 @@
<li class="third-level"><a
href="#table-metadata-and-snapshots">Table Metadata and Snapshots</a></li>
<li class="second-level"><a
href="#appendix-d-single-value-serialization">Appendix D: Single-value
serialization</a></li>
+ <li class="second-level"><a href="#format-version-changes">Format
version changes</a></li>
+
+ <li class="third-level"><a href="#version-2">Version 2</a></li>
</ul>
</div></div>
<div class="col-md-9" role="main">
@@ -691,14 +701,14 @@ Timestamps <em>without time zone</em> represent a date
and time of day regardles
<td><strong>Deprecated. Always write a default value and do not
read.</strong></td>
</tr>
<tr>
-<td><strong><code>106 file_ordinal</code></strong></td>
+<td><del><strong><code>106 file_ordinal</code></strong></del></td>
<td><code>optional int</code></td>
-<td>Ordinal of the file w.r.t files with the same partition tuple and snapshot
id</td>
+<td><strong>Deprecated. Do not use.</strong></td>
</tr>
<tr>
-<td><strong><code>107 sort_columns</code></strong></td>
+<td><del><strong><code>107 sort_columns</code></strong></del></td>
<td><code>optional list</code></td>
-<td>Columns the file is sorted by</td>
+<td><strong>Deprecated. Do not use.</strong></td>
</tr>
<tr>
<td><strong><code>108 column_sizes</code></strong></td>
@@ -761,6 +771,7 @@ Timestamps <em>without time zone</em> represent a date and
time of day regardles
<ul>
<li><strong><code>snapshot-id</code></strong> – A unique long ID.</li>
<li><strong><code>parent-snapshot-id</code></strong> – (Optional) The
snapshot ID of the snapshot’s parent. This field is not present for snapshots
that have no parent snapshot, such as snapshots created before this field was
added or the first snapshot of a table.</li>
+<li><strong><code>sequence-number</code></strong> – A monotonically
increasing long that tracks the order of snapshots in a table. (<strong>v2
only</strong>)</li>
<li><strong><code>timestamp-ms</code></strong> – A timestamp when the
snapshot was created. This is used when garbage collecting snapshots.</li>
<li><strong><code>manifests</code></strong> – A list of manifest file
locations. The data files in a snapshot are the union of all data files listed
in these manifests. (Deprecated in favor of <code>manifest-list</code>)</li>
<li><strong><code>manifest-list</code></strong> – (Optional) The
location of a manifest list file for this snapshot, which contains a list of
manifest files with additional metadata. If present, the manifests field must
be omitted.</li>
@@ -841,6 +852,21 @@ Timestamps <em>without time zone</em> represent a date and
time of day regardles
<td><code>list<508: field_summary></code> (see below)</td>
<td>A list of field summaries for each partition field in the spec. Each field
in the list corresponds to a field in the manifest file’s partition spec.</td>
</tr>
+<tr>
+<td><strong><code>512 added_rows_count</code></strong></td>
+<td><code>long</code></td>
+<td>Number of rows in all of files in the manifest that have status
<code>ADDED</code></td>
+</tr>
+<tr>
+<td><strong><code>513 existing_rows_count</code></strong></td>
+<td><code>long</code></td>
+<td>Number of rows in all of files in the manifest that have status
<code>EXISTING</code></td>
+</tr>
+<tr>
+<td><strong><code>514 deleted_rows_count</code></strong></td>
+<td><code>long</code></td>
+<td>Number of rows in all of files in the manifest that have status
<code>DELETED</code></td>
+</tr>
</tbody>
</table>
<p><code>field_summary</code> is a struct with the following fields</p>
@@ -877,6 +903,38 @@ Timestamps <em>without time zone</em> represent a date and
time of day regardles
<h3 id="table-metadata">Table Metadata<a class="headerlink"
href="#table-metadata" title="Permanent link">¶</a></h3>
<p>Table metadata is stored as JSON. Each table metadata change creates a new
table metadata file that is committed by an atomic operation. This operation is
used to ensure that a new version of table metadata replaces the version on
which it was based. This produces a linear history of table versions and
ensures that concurrent writes are not lost.</p>
<p>The atomic operation used to commit metadata depends on how tables are
tracked and is not standardized by this spec. See the sections below for
examples.</p>
+<h3 id="delete-format">Delete Format<a class="headerlink"
href="#delete-format" title="Permanent link">¶</a></h3>
+<p>This section details how to encode row-level deletes in Iceberg metadata.
Row-level deletes are not supported in the current format version 1. This part
of the spec is not yet complete and will be completed as format version 2.</p>
+<h4 id="position-based-delete-files">Position-based Delete Files<a
class="headerlink" href="#position-based-delete-files" title="Permanent
link">¶</a></h4>
+<p>Position-based delete files identify rows in one or more data files that
have been deleted.</p>
+<p>Position-based delete files store <code>file_position_delete</code>, a
struct with the following fields:</p>
+<table>
+<thead>
+<tr>
+<th>Field id, name</th>
+<th>Type</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>1 file_path</code></strong></td>
+<td><code>required string</code></td>
+<td>The full URI of a data file with FS scheme. This must match the
<code>file_path</code> of the target data file in a manifest entry.</td>
+</tr>
+<tr>
+<td><strong><code>2 position</code></strong></td>
+<td><code>required long</code></td>
+<td>The ordinal position of a deleted row in the target data file identified
by <code>file_path</code>, starting at <code>0</code>.</td>
+</tr>
+</tbody>
+</table>
+<p>The rows in the delete file must be sorted by <code>file_path</code> then
<code>position</code> to optimize filtering rows while scanning. </p>
+<ul>
+<li>Sorting by <code>file_path</code> allows filter pushdown by file in
columnar storage formats.</li>
+<li>Sorting by <code>position</code> allows filtering rows while scanning, to
avoid keeping deletes in memory.</li>
+</ul>
+<p>Though the delete files can be written using any supported data file format
in Iceberg, it is recommended to write delete files with same file format as
the table’s file format.</p>
<h4 id="commit-conflict-resolution-and-retry">Commit Conflict Resolution and
Retry<a class="headerlink" href="#commit-conflict-resolution-and-retry"
title="Permanent link">¶</a></h4>
<p>When two commits happen at the same time and are based on the same version,
only one commit will succeed. In most cases, the failed commit can be applied
to the new current version of table metadata and retried. Updates verify the
conditions under which they can be applied to a new version and retry if those
conditions are met.</p>
<ul>
@@ -890,60 +948,94 @@ Timestamps <em>without time zone</em> represent a date
and time of day regardles
<table>
<thead>
<tr>
+<th>Format v1</th>
+<th>Format v2</th>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
+<td><em>required</em></td>
+<td><em>required</em></td>
<td><strong><code>format-version</code></strong></td>
<td>An integer version number for the format. Currently, this is always 1.
Implementations must throw an exception if a table’s version is higher
than the supported version.</td>
</tr>
<tr>
+<td><em>optional</em></td>
+<td><em>required</em></td>
<td><strong><code>table-uuid</code></strong></td>
<td>A UUID that identifies the table, generated when the table is created.
Implementations must throw an exception if a table’s UUID does not match
the expected UUID after refreshing metadata.</td>
</tr>
<tr>
+<td><em>required</em></td>
+<td><em>required</em></td>
<td><strong><code>location</code></strong></td>
-<td>The table’s base location. This is used by writers to determine where to
store data files, manifest files, and table metadata files.</td>
+<td>The table’s base location. This is used by writers to determine
where to store data files, manifest files, and table metadata files.</td>
</tr>
<tr>
+<td><em>omitted</em></td>
+<td><em>required</em></td>
+<td><strong><code>sequence-number</code></strong></td>
+<td>The table’s highest assigned sequence number, a monotonically
increasing long that tracks the order of snapshots in a table.</td>
+</tr>
+<tr>
+<td><em>required</em></td>
+<td><em>required</em></td>
<td><strong><code>last-updated-ms</code></strong></td>
<td>Timestamp in milliseconds from the unix epoch when the table was last
updated. Each table metadata file should update this field just before
writing.</td>
</tr>
<tr>
+<td><em>required</em></td>
+<td><em>required</em></td>
<td><strong><code>last-column-id</code></strong></td>
<td>An integer; the highest assigned column ID for the table. This is used to
ensure columns are always assigned an unused ID when evolving schemas.</td>
</tr>
<tr>
+<td><em>required</em></td>
+<td><em>required</em></td>
<td><strong><code>schema</code></strong></td>
<td>The table’s current schema.</td>
</tr>
<tr>
+<td><em>required</em></td>
+<td><em>omitted</em></td>
<td><strong><code>partition-spec</code></strong></td>
<td>The table’s current partition spec, stored as only fields. Note that this
is used by writers to partition data, but is not used when reading because
reads use the specs stored in manifest files. (<strong>Deprecated</strong>: use
<code>partition-specs</code> and <code>default-spec-id</code>instead )</td>
</tr>
<tr>
+<td><em>optional</em></td>
+<td><em>required</em></td>
<td><strong><code>partition-specs</code></strong></td>
<td>A list of partition specs, stored as full partition spec objects.</td>
</tr>
<tr>
+<td><em>optional</em></td>
+<td><em>required</em></td>
<td><strong><code>default-spec-id</code></strong></td>
<td>ID of the “current” spec that writers should use by default.</td>
</tr>
<tr>
+<td><em>optional</em></td>
+<td><em>optional</em></td>
<td><strong><code>properties</code></strong></td>
<td>A string to string map of table properties. This is used to control
settings that affect reading and writing and is not intended to be used for
arbitrary metadata. For example, <code>commit.retry.num-retries</code> is used
to control the number of commit retries.</td>
</tr>
<tr>
+<td><em>optional</em></td>
+<td><em>optional</em></td>
<td><strong><code>current-snapshot-id</code></strong></td>
<td><code>long</code> ID of the current table snapshot.</td>
</tr>
<tr>
+<td><em>optional</em></td>
+<td><em>optional</em></td>
<td><strong><code>snapshots</code></strong></td>
<td>A list of valid snapshots. Valid snapshots are snapshots for which all
data files exist in the file system. A data file must not be deleted from the
file system until the last snapshot in which it was listed is garbage
collected.</td>
</tr>
<tr>
+<td><em>optional</em></td>
+<td><em>optional</em></td>
<td><strong><code>snapshot-log</code></strong></td>
<td>A list (optional) of timestamp and snapshot ID pairs that encodes changes
to the current snapshot for the table. Each time the current-snapshot-id is
changed, a new entry should be added with the last-updated-ms and the new
current-snapshot-id. When snapshots are expired from the list of valid
snapshots, all entries before a snapshot that has expired should be
removed.</td>
</tr>
@@ -1257,6 +1349,7 @@ Timestamps <em>without time zone</em> represent a date
and time of day regardles
<tr>
<th>Type</th>
<th>ORC type</th>
+<th>ORC type attributes</th>
<th>Notes</th>
</tr>
</thead>
@@ -1265,91 +1358,112 @@ Timestamps <em>without time zone</em> represent a date
and time of day regardles
<td><strong><code>boolean</code></strong></td>
<td><code>boolean</code></td>
<td></td>
+<td></td>
</tr>
<tr>
<td><strong><code>int</code></strong></td>
<td><code>int</code></td>
+<td></td>
<td>ORC <code>tinyint</code> and <code>smallint</code> would also map to
<strong><code>int</code></strong>.</td>
</tr>
<tr>
<td><strong><code>long</code></strong></td>
<td><code>long</code></td>
<td></td>
+<td></td>
</tr>
<tr>
<td><strong><code>float</code></strong></td>
<td><code>float</code></td>
<td></td>
+<td></td>
</tr>
<tr>
<td><strong><code>double</code></strong></td>
<td><code>double</code></td>
<td></td>
+<td></td>
</tr>
<tr>
<td><strong><code>decimal(P,S)</code></strong></td>
<td><code>decimal</code></td>
<td></td>
+<td></td>
</tr>
<tr>
<td><strong><code>date</code></strong></td>
<td><code>date</code></td>
<td></td>
+<td></td>
</tr>
<tr>
<td><strong><code>time</code></strong></td>
-<td><code>int</code></td>
+<td><code>long</code></td>
+<td><code>iceberg.long-type</code>=<code>TIME</code></td>
<td>Stores microseconds from midnight.</td>
</tr>
<tr>
<td><strong><code>timestamp</code></strong></td>
<td><code>timestamp</code></td>
<td></td>
+<td>[1]</td>
</tr>
<tr>
<td><strong><code>timestamptz</code></strong></td>
-<td><code>struct</code></td>
-<td>We should add this to ORC’s type model (ORC-294).</td>
+<td><code>timestamp_instant</code></td>
+<td></td>
+<td>[1]</td>
</tr>
<tr>
<td><strong><code>string</code></strong></td>
<td><code>string</code></td>
+<td></td>
<td>ORC <code>varchar</code> and <code>char</code> would also map to
<strong><code>string</code></strong>.</td>
</tr>
<tr>
<td><strong><code>uuid</code></strong></td>
<td><code>binary</code></td>
+<td><code>iceberg.binary-type</code>=<code>UUID</code></td>
<td></td>
</tr>
<tr>
<td><strong><code>fixed(L)</code></strong></td>
<td><code>binary</code></td>
+<td><code>iceberg.binary-type</code>=<code>FIXED</code> &
<code>iceberg.length</code>=<code>L</code></td>
<td>The length would not be checked by the ORC reader and should be checked by
the adapter.</td>
</tr>
<tr>
<td><strong><code>binary</code></strong></td>
<td><code>binary</code></td>
<td></td>
+<td></td>
</tr>
<tr>
<td><strong><code>struct</code></strong></td>
<td><code>struct</code></td>
-<td>ORC <code>uniontype</code> would also map to
<strong><code>struct</code></strong>.</td>
+<td></td>
+<td></td>
</tr>
<tr>
<td><strong><code>list</code></strong></td>
<td><code>array</code></td>
<td></td>
+<td></td>
</tr>
<tr>
<td><strong><code>map</code></strong></td>
<td><code>map</code></td>
<td></td>
+<td></td>
</tr>
</tbody>
</table>
+<p>Notes:</p>
+<ol>
+<li>ORC’s <a
href="https://orc.apache.org/api/hive-storage-api/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.html";>TimestampColumnVector</a>
comprises of a time field (milliseconds since epoch) and a nanos field
(nanoseconds within the second). Hence the milliseconds within the second are
reported twice; once in the time field and again in the nanos field. The read
adapter should only use milliseconds within the second from one of these
fields. The write adapter shoul [...]
+</ol>
<p>One of the interesting challenges with this is how to map Iceberg’s schema
evolution (id based) on to ORC’s (name based). In theory, we could use
Iceberg’s column ids as the column and field names, but that would suck from a
user’s point of view. </p>
-<p>The column ids would be stored in ORC’s user metadata as
<code>iceberg.column.id</code> with a comma separated list of the ids.</p>
+<p>The column IDs must be stored in ORC type attributes using the key
<code>iceberg.id</code>, and <code>iceberg.required</code> to store
<code>"true"</code> if the Iceberg column is required, otherwise it will be
optional.</p>
<p>Iceberg would build the desired reader schema with their schema evolution
rules and pass that down to the ORC reader, which would then use its schema
evolution to map that to the writer’s schema. Basically, Iceberg would need to
change the names of columns and fields to get the desired mapping.</p>
<table>
<thead>
@@ -1811,7 +1925,25 @@ Hash results are not dependent on decimal scale, which
is part of the type, not
<td>Not supported</td>
</tr>
</tbody>
-</table></div>
+</table>
+<h2 id="format-version-changes">Format version changes<a class="headerlink"
href="#format-version-changes" title="Permanent link">¶</a></h2>
+<h3 id="version-2">Version 2<a class="headerlink" href="#version-2"
title="Permanent link">¶</a></h3>
+<p>Writing v1 metadata:
+* Table metadata field <code>last-sequence-number</code> should not be written.
+* Snapshot field <code>sequence-number</code> should not be written.</p>
+<p>Reading v1 metadata:
+* Table metadata field <code>last-sequence-number</code> must default to 0.
+* Snapshot field <code>sequence-number</code> must default to 0.</p>
+<p>Writing v2 metadata:
+* Table metadata added required field <code>last-sequence-number</code>.
+* Table metadata now requires field <code>table-uuid</code>.
+* Table metadata now requires field <code>partition-specs</code>.
+* Table metadata now requires field <code>default-spec-id</code>.
+* Table metadata field <code>partition-spec</code> is no longer required and
may be omitted.
+* Snapshot added required field field <code>sequence-number</code>.
+* Snapshot now requires field <code>manifest-list</code>.
+* Snapshot field <code>manifests</code> is no longer allowed.</p>
+<p>Note that these requirements apply when writing data to a v2 table. Tables
that are upgraded from v1 may contain metadata that does not follow these
requirements. Implementations should remain backward-compatible with v1
metadata requirements.</p></div>
</div>
diff --git a/terms/index.html b/terms/index.html
index 4bb4b1a..36e6624 100644
--- a/terms/index.html
+++ b/terms/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
diff --git a/why-iceberg/index.html b/why-iceberg/index.html
index 2411fc3..027b463 100644
--- a/why-iceberg/index.html
+++ b/why-iceberg/index.html
@@ -116,19 +116,25 @@
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="../getting-started/">Getting Started</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="../configuration/">Configuration</a>
</li>
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="../schemas/">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -152,13 +158,13 @@
<li >
- <a href="../spark#time-travel">Time travel</a>
+ <a href="../spark#time-travel">Time Travel</a>
</li>
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../api-quickstart/">Spark Quickstart</a>
</li>
@@ -191,7 +197,7 @@
<li >
- <a href="../api-quickstart/">Quickstart</a>
+ <a href="../java-api-quickstart/">Quickstart</a>
</li>
@@ -236,7 +242,7 @@
<li >
- <a href="../python-api-intro/">API intro</a>
+ <a href="../python-api-intro/">API Intro</a>
</li>
