This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/asf-site by this push:
new b353b0b Travis CI build asf-site
b353b0b is described below
commit b353b0b66cf17ccfbbf55d58d741fdebd66ac0a4
Author: CI <[email protected]>
AuthorDate: Fri Apr 9 08:39:56 2021 +0000
Travis CI build asf-site
---
content/docs/0.8.0-concurrency_control.html | 6 ------
content/docs/0.8.0-configurations.html | 32 ++++++++++++++++-------------
content/docs/concurrency_control.html | 6 ------
content/docs/configurations.html | 32 ++++++++++++++++-------------
4 files changed, 36 insertions(+), 40 deletions(-)
diff --git a/content/docs/0.8.0-concurrency_control.html
b/content/docs/0.8.0-concurrency_control.html
index d28ec87..4e540b5 100644
--- a/content/docs/0.8.0-concurrency_control.html
+++ b/content/docs/0.8.0-concurrency_control.html
@@ -403,8 +403,6 @@ hoodie.write.lock.provider=<lock-provider-classname>
<div class="highlighter-rouge"><div class="highlight"><pre
class="highlight"><code>hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider
hoodie.write.lock.zookeeper.url
hoodie.write.lock.zookeeper.port
-hoodie.write.lock.wait_time_ms
-hoodie.write.lock.num_retries
hoodie.write.lock.zookeeper.lock_key
hoodie.write.lock.zookeeper.base_path
</code></pre></div></div>
@@ -414,8 +412,6 @@ hoodie.write.lock.zookeeper.base_path
<div class="highlighter-rouge"><div class="highlight"><pre
class="highlight"><code>hoodie.write.lock.provider=org.apache.hudi.hive.HiveMetastoreBasedLockProvider
hoodie.write.lock.hivemetastore.database
hoodie.write.lock.hivemetastore.table
-hoodie.write.lock.wait_time_ms
-hoodie.write.lock.num_retries
</code></pre></div></div>
<p><code class="highlighter-rouge">The HiveMetastore URI's are picked up from
the hadoop configuration file loaded during runtime.</code></p>
@@ -433,8 +429,6 @@ hoodie.write.lock.num_retries
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="s">"hoodie.write.concurrency.mode"</span><span
class="o">,</span> <span class="s">"optimistic_concurrency_control"</span><span
class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="s">"hoodie.write.lock.zookeeper.url"</span><span
class="o">,</span> <span class="s">"zookeeper"</span><span class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span
class="s">"hoodie.write.lock.zookeeper.port"</span><span class="o">,</span>
<span class="s">"2181"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="s">"hoodie.write.lock.wait_time_ms"</span><span
class="o">,</span> <span class="s">"12000"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="s">"hoodie.write.lock.num_retries"</span><span
class="o">,</span> <span class="s">"2"</span><span class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span
class="s">"hoodie.write.lock.zookeeper.lock_key"</span><span class="o">,</span>
<span class="s">"test_table"</span><span class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span
class="s">"hoodie.write.lock.zookeeper.base_path"</span><span
class="o">,</span> <span class="s">"/test"</span><span class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="no">RECORDKEY_FIELD_OPT_KEY</span><span
class="o">,</span> <span class="s">"uuid"</span><span class="o">)</span>
diff --git a/content/docs/0.8.0-configurations.html
b/content/docs/0.8.0-configurations.html
index 434418f..960dfb4 100644
--- a/content/docs/0.8.0-configurations.html
+++ b/content/docs/0.8.0-configurations.html
@@ -999,6 +999,10 @@ HoodieWriteConfig can be built using a builder pattern as
below.</p>
<p>Property: <code class="highlighter-rouge">hoodie.cleaner.policy</code> <br
/>
<span style="color:grey"> Cleaning policy to be used. Hudi will delete older
versions of parquet files to re-claim space. Any Query/Computation referring to
this version of the file will fail. It is good to make sure that the data is
retained for more than the maximum query execution time.</span></p>
+<h4 id="withFailedWritesCleaningPolicy">withFailedWritesCleaningPolicy(policy
= HoodieFailedWritesCleaningPolicy.EAGER)</h4>
+<p>Property: <code
class="highlighter-rouge">hoodie.cleaner.policy.failed.writes</code> <br />
+<span style="color:grey"> Cleaning policy for failed writes to be used. Hudi
will delete any files written by failed writes to re-claim space. Choose to
perform this rollback of failed writes <code
class="highlighter-rouge">eagerly</code> before every writer starts (only
supported for single writer) or <code class="highlighter-rouge">lazily</code>
by the cleaner (required for multi-writers)</span></p>
+
<h4 id="retainCommits">retainCommits(no_of_commits_to_retain = 24)</h4>
<p>Property: <code
class="highlighter-rouge">hoodie.cleaner.commits.retained</code> <br />
<span style="color:grey">Number of commits to retain. So data will be retained
for num_of_commits * time_between_commits (scheduled). This also directly
translates into how much you can incrementally pull on this table</span></p>
@@ -1360,59 +1364,59 @@ Each clustering operation can create multiple groups.
Total amount of data proce
<a href="#withLockConfig">withLockConfig</a> (HoodieLockConfig) <br /></p>
<h4 id="withLockProvider">withLockProvider(lockProvider =
org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.provider</code> <br />
+<p>Property: <code class="highlighter-rouge">hoodie.write.lock.provider</code>
<br />
<span style="color:grey">Lock provider class name, user can provide their own
implementation of LockProvider which should be subclass of
org.apache.hudi.common.lock.LockProvider</span></p>
<h4 id="withZkQuorum">withZkQuorum(zkQuorum)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.url</code> <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.url</code> <br />
<span style="color:grey">Set the list of comma separated servers to connect
to</span></p>
<h4 id="withZkBasePath">withZkBasePath(zkBasePath)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.base_path</code>
[Required] <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.base_path</code>
[Required] <br />
<span style="color:grey">The base path on Zookeeper under which to create a
ZNode to acquire the lock. This should be common for all jobs writing to the
same table</span></p>
<h4 id="withZkPort">withZkPort(zkPort)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.port</code> [Required]
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.port</code> [Required]
<br />
<span style="color:grey">The connection port to be used for
Zookeeper</span></p>
<h4 id="withZkLockKey">withZkLockKey(zkLockKey)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.lock_key</code>
[Required] <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.lock_key</code>
[Required] <br />
<span style="color:grey">Key name under base_path at which to create a ZNode
and acquire lock. Final path on zk will look like base_path/lock_key. We
recommend setting this to the table name</span></p>
<h4
id="withZkConnectionTimeoutInMs">withZkConnectionTimeoutInMs(connectionTimeoutInMs
= 15000)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.connection_timeout_ms</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.connection_timeout_ms</code>
<br />
<span style="color:grey">How long to wait when connecting to ZooKeeper before
considering the connection a failure</span></p>
<h4 id="withZkSessionTimeoutInMs">withZkSessionTimeoutInMs(sessionTimeoutInMs
= 60000)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.session_timeout_ms</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.session_timeout_ms</code>
<br />
<span style="color:grey">How long to wait after losing a connection to
ZooKeeper before the session is expired</span></p>
<h4 id="withNumRetries">withNumRetries(num_retries = 3)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.num_retries</code> <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.num_retries</code> <br />
<span style="color:grey">Maximum number of times to retry by lock provider
client</span></p>
<h4
id="withRetryWaitTimeInMillis">withRetryWaitTimeInMillis(retryWaitTimeInMillis
= 5000)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.wait_time_ms_between_retry</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.wait_time_ms_between_retry</code>
<br />
<span style="color:grey">Initial amount of time to wait between retries by
lock provider client</span></p>
<h4 id="withHiveDatabaseName">withHiveDatabaseName(hiveDatabaseName)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.hivemetastore.database</code>
[Required] <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.hivemetastore.database</code>
[Required] <br />
<span style="color:grey">The Hive database to acquire lock against</span></p>
<h4 id="withHiveTableName">withHiveTableName(hiveTableName)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.hivemetastore.table</code>
[Required] <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.hivemetastore.table</code>
[Required] <br />
<span style="color:grey">The Hive table under the hive database to acquire
lock against</span></p>
<h4 id="withClientNumRetries">withClientNumRetries(clientNumRetries = 0)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.client.num_retries</code> <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.client.num_retries</code> <br />
<span style="color:grey">Maximum number of times to retry to acquire lock
additionally from the hudi client</span></p>
<h4
id="withRetryWaitTimeInMillis">withRetryWaitTimeInMillis(retryWaitTimeInMillis
= 10000)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.client.wait_time_ms_between_retry</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.client.wait_time_ms_between_retry</code>
<br />
<span style="color:grey">Amount of time to wait between retries from the hudi
client</span></p>
<h4
id="withConflictResolutionStrategy">withConflictResolutionStrategy(lockProvider
=
org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.conflict.resolution.strategy</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.conflict.resolution.strategy</code>
<br />
<span style="color:grey">Lock provider class name, this should be subclass of
org.apache.hudi.client.transaction.ConflictResolutionStrategy</span></p>
diff --git a/content/docs/concurrency_control.html
b/content/docs/concurrency_control.html
index 214da87..2353ffb 100644
--- a/content/docs/concurrency_control.html
+++ b/content/docs/concurrency_control.html
@@ -425,8 +425,6 @@ hoodie.write.lock.provider=<lock-provider-classname>
<div class="highlighter-rouge"><div class="highlight"><pre
class="highlight"><code>hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider
hoodie.write.lock.zookeeper.url
hoodie.write.lock.zookeeper.port
-hoodie.write.lock.wait_time_ms
-hoodie.write.lock.num_retries
hoodie.write.lock.zookeeper.lock_key
hoodie.write.lock.zookeeper.base_path
</code></pre></div></div>
@@ -436,8 +434,6 @@ hoodie.write.lock.zookeeper.base_path
<div class="highlighter-rouge"><div class="highlight"><pre
class="highlight"><code>hoodie.write.lock.provider=org.apache.hudi.hive.HiveMetastoreBasedLockProvider
hoodie.write.lock.hivemetastore.database
hoodie.write.lock.hivemetastore.table
-hoodie.write.lock.wait_time_ms
-hoodie.write.lock.num_retries
</code></pre></div></div>
<p><code class="highlighter-rouge">The HiveMetastore URI's are picked up from
the hadoop configuration file loaded during runtime.</code></p>
@@ -455,8 +451,6 @@ hoodie.write.lock.num_retries
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="s">"hoodie.write.concurrency.mode"</span><span
class="o">,</span> <span class="s">"optimistic_concurrency_control"</span><span
class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="s">"hoodie.write.lock.zookeeper.url"</span><span
class="o">,</span> <span class="s">"zookeeper"</span><span class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span
class="s">"hoodie.write.lock.zookeeper.port"</span><span class="o">,</span>
<span class="s">"2181"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="s">"hoodie.write.lock.wait_time_ms"</span><span
class="o">,</span> <span class="s">"12000"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="s">"hoodie.write.lock.num_retries"</span><span
class="o">,</span> <span class="s">"2"</span><span class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span
class="s">"hoodie.write.lock.zookeeper.lock_key"</span><span class="o">,</span>
<span class="s">"test_table"</span><span class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span
class="s">"hoodie.write.lock.zookeeper.base_path"</span><span
class="o">,</span> <span class="s">"/test"</span><span class="o">)</span>
<span class="o">.</span><span class="na">option</span><span
class="o">(</span><span class="no">RECORDKEY_FIELD_OPT_KEY</span><span
class="o">,</span> <span class="s">"uuid"</span><span class="o">)</span>
diff --git a/content/docs/configurations.html b/content/docs/configurations.html
index 99bd6f3..2ea0368 100644
--- a/content/docs/configurations.html
+++ b/content/docs/configurations.html
@@ -1021,6 +1021,10 @@ HoodieWriteConfig can be built using a builder pattern
as below.</p>
<p>Property: <code class="highlighter-rouge">hoodie.cleaner.policy</code> <br
/>
<span style="color:grey"> Cleaning policy to be used. Hudi will delete older
versions of parquet files to re-claim space. Any Query/Computation referring to
this version of the file will fail. It is good to make sure that the data is
retained for more than the maximum query execution time.</span></p>
+<h4 id="withFailedWritesCleaningPolicy">withFailedWritesCleaningPolicy(policy
= HoodieFailedWritesCleaningPolicy.EAGER)</h4>
+<p>Property: <code
class="highlighter-rouge">hoodie.cleaner.policy.failed.writes</code> <br />
+<span style="color:grey"> Cleaning policy for failed writes to be used. Hudi
will delete any files written by failed writes to re-claim space. Choose to
perform this rollback of failed writes <code
class="highlighter-rouge">eagerly</code> before every writer starts (only
supported for single writer) or <code class="highlighter-rouge">lazily</code>
by the cleaner (required for multi-writers)</span></p>
+
<h4 id="retainCommits">retainCommits(no_of_commits_to_retain = 24)</h4>
<p>Property: <code
class="highlighter-rouge">hoodie.cleaner.commits.retained</code> <br />
<span style="color:grey">Number of commits to retain. So data will be retained
for num_of_commits * time_between_commits (scheduled). This also directly
translates into how much you can incrementally pull on this table</span></p>
@@ -1382,59 +1386,59 @@ Each clustering operation can create multiple groups.
Total amount of data proce
<a href="#withLockConfig">withLockConfig</a> (HoodieLockConfig) <br /></p>
<h4 id="withLockProvider">withLockProvider(lockProvider =
org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.provider</code> <br />
+<p>Property: <code class="highlighter-rouge">hoodie.write.lock.provider</code>
<br />
<span style="color:grey">Lock provider class name, user can provide their own
implementation of LockProvider which should be subclass of
org.apache.hudi.common.lock.LockProvider</span></p>
<h4 id="withZkQuorum">withZkQuorum(zkQuorum)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.url</code> <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.url</code> <br />
<span style="color:grey">Set the list of comma separated servers to connect
to</span></p>
<h4 id="withZkBasePath">withZkBasePath(zkBasePath)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.base_path</code>
[Required] <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.base_path</code>
[Required] <br />
<span style="color:grey">The base path on Zookeeper under which to create a
ZNode to acquire the lock. This should be common for all jobs writing to the
same table</span></p>
<h4 id="withZkPort">withZkPort(zkPort)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.port</code> [Required]
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.port</code> [Required]
<br />
<span style="color:grey">The connection port to be used for
Zookeeper</span></p>
<h4 id="withZkLockKey">withZkLockKey(zkLockKey)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.lock_key</code>
[Required] <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.lock_key</code>
[Required] <br />
<span style="color:grey">Key name under base_path at which to create a ZNode
and acquire lock. Final path on zk will look like base_path/lock_key. We
recommend setting this to the table name</span></p>
<h4
id="withZkConnectionTimeoutInMs">withZkConnectionTimeoutInMs(connectionTimeoutInMs
= 15000)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.connection_timeout_ms</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.connection_timeout_ms</code>
<br />
<span style="color:grey">How long to wait when connecting to ZooKeeper before
considering the connection a failure</span></p>
<h4 id="withZkSessionTimeoutInMs">withZkSessionTimeoutInMs(sessionTimeoutInMs
= 60000)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.zookeeper.session_timeout_ms</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.zookeeper.session_timeout_ms</code>
<br />
<span style="color:grey">How long to wait after losing a connection to
ZooKeeper before the session is expired</span></p>
<h4 id="withNumRetries">withNumRetries(num_retries = 3)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.num_retries</code> <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.num_retries</code> <br />
<span style="color:grey">Maximum number of times to retry by lock provider
client</span></p>
<h4
id="withRetryWaitTimeInMillis">withRetryWaitTimeInMillis(retryWaitTimeInMillis
= 5000)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.wait_time_ms_between_retry</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.wait_time_ms_between_retry</code>
<br />
<span style="color:grey">Initial amount of time to wait between retries by
lock provider client</span></p>
<h4 id="withHiveDatabaseName">withHiveDatabaseName(hiveDatabaseName)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.hivemetastore.database</code>
[Required] <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.hivemetastore.database</code>
[Required] <br />
<span style="color:grey">The Hive database to acquire lock against</span></p>
<h4 id="withHiveTableName">withHiveTableName(hiveTableName)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.hivemetastore.table</code>
[Required] <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.hivemetastore.table</code>
[Required] <br />
<span style="color:grey">The Hive table under the hive database to acquire
lock against</span></p>
<h4 id="withClientNumRetries">withClientNumRetries(clientNumRetries = 0)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.client.num_retries</code> <br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.client.num_retries</code> <br />
<span style="color:grey">Maximum number of times to retry to acquire lock
additionally from the hudi client</span></p>
<h4
id="withRetryWaitTimeInMillis">withRetryWaitTimeInMillis(retryWaitTimeInMillis
= 10000)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.client.wait_time_ms_between_retry</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.client.wait_time_ms_between_retry</code>
<br />
<span style="color:grey">Amount of time to wait between retries from the hudi
client</span></p>
<h4
id="withConflictResolutionStrategy">withConflictResolutionStrategy(lockProvider
=
org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy)</h4>
-<p>Property: <code
class="highlighter-rouge">hoodie.writer.lock.conflict.resolution.strategy</code>
<br />
+<p>Property: <code
class="highlighter-rouge">hoodie.write.lock.conflict.resolution.strategy</code>
<br />
<span style="color:grey">Lock provider class name, this should be subclass of
org.apache.hudi.client.transaction.ConflictResolutionStrategy</span></p>
