This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/asf-site by this push:
new d2d133a Travis CI build asf-site
d2d133a is described below
commit d2d133a2953867b9557f365697b235605d979009
Author: CI <[email protected]>
AuthorDate: Wed May 27 14:58:11 2020 +0000
Travis CI build asf-site
---
content/assets/js/lunr/lunr-store.js | 14 +-
.../oss_hoodie.html => cn/docs/azure_hoodie.html} | 157 ++++++++++-----------
content/cn/docs/configurations.html | 4 +
content/{ => cn}/docs/oss_hoodie.html | 66 ++++-----
.../docs/{oss_hoodie.html => azure_hoodie.html} | 115 +++++++--------
content/docs/configurations.html | 4 +
content/docs/oss_hoodie.html | 72 +++++-----
content/sitemap.xml | 10 +-
8 files changed, 219 insertions(+), 223 deletions(-)
diff --git a/content/assets/js/lunr/lunr-store.js
b/content/assets/js/lunr/lunr-store.js
index d133410..e7b9308 100644
--- a/content/assets/js/lunr/lunr-store.js
+++ b/content/assets/js/lunr/lunr-store.js
@@ -537,13 +537,23 @@ var store = [{
"title": "OSS Filesystem",
"excerpt":"这个页面描述了如何让你的Hudi spark任务使用Aliyun OSS存储。 Aliyun OSS 部署
为了让Hudi使用OSS，需要增加两部分的配置: 为Hidi增加Aliyun OSS的相关配置 增加Jar包的MVN依赖 Aliyun OSS 相关的配置
新增下面的配置到你的Hudi能访问的core-site.xml文件。使用你的OSS bucket name替换掉fs.defaultFS，使用OSS
endpoint地址替换fs.oss.endpoint，使用OSS的key和secret分别替换fs.oss.accessKeyId和fs.oss.accessKeySecret。主要Hudi就能读写相应的bucket。
<property> <name>fs.defaultFS</name>
<value>oss://bucketname/</value> </property> <property>
<name>fs.oss.e [...]
"tags": [],
- "url": "https://hudi.apache.org/docs/oss_hoodie.html";,
+ "url": "https://hudi.apache.org/cn/docs/oss_hoodie.html";,
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
"title": "OSS Filesystem",
"excerpt":"In this page, we explain how to get your Hudi spark job to
store into Aliyun OSS. Aliyun OSS configs There are two configurations required
for Hudi-OSS compatibility: Adding Aliyun OSS Credentials for Hudi Adding
required Jars to classpath Aliyun OSS Credentials Add the required configs in
your core-site.xml from...","categories": [],
"tags": [],
"url": "https://hudi.apache.org/docs/oss_hoodie.html";,
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
+ "title": "Azure Filesystem",
+ "excerpt":"In this page, we explain how to use Hudi on Microsoft
Azure. Disclaimer This page is maintained by the Hudi community. If the
information is inaccurate or you have additional information to add. Please
feel free to create a JIRA ticket. Contribution is highly appreciated.
Supported Storage System There are...","categories": [],
+ "tags": [],
+ "url": "https://hudi.apache.org/cn/docs/azure_hoodie.html";,
+ "teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
+ "title": "Azure Filesystem",
+ "excerpt":"In this page, we explain how to use Hudi on Microsoft
Azure. Disclaimer This page is maintained by the Hudi community. If the
information is inaccurate or you have additional information to add. Please
feel free to create a JIRA ticket. Contribution is highly appreciated.
Supported Storage System There are...","categories": [],
+ "tags": [],
+ "url": "https://hudi.apache.org/docs/azure_hoodie.html";,
+ "teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
"title": "Quick-Start Guide",
"excerpt":"本指南通过使用spark-shell简要介绍了Hudi功能。使用Spark数据源，我们将通过代码段展示如何插入和更新Hudi的默认存储类型数据集：
写时复制。每次写操作之后，我们还将展示如何读取快照和增量数据。 设置spark-shell
Hudi适用于Spark-2.x版本。您可以按照此处的说明设置spark。 在提取的目录中，使用spark-shell运行Hudi：
bin/spark-shell --packages org.apache.hudi:hudi-spark-bundle:0.5.0-incubating
--conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer'
设置表名、基本路径和数据生成器来为本指南生成记录。 import org.apache.hudi.QuickstartUtils._ import
scala.collection.JavaConversions._ import org.apache.spark.sql.Sa [...]
"tags": [],
@@ -620,7 +630,7 @@ var store = [{
"url": "https://hudi.apache.org/docs/querying_data.html";,
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
"title": "配置",
- "excerpt":"该页面介绍了几种配置写入或读取Hudi数据集的作业的方法。 简而言之，您可以在几个级别上控制行为。
Spark数据源配置 : 这些配置控制Hudi Spark数据源，提供如下功能： 定义键和分区、选择写操作、指定如何合并记录或选择要读取的视图类型。
WriteClient 配置 : 在内部，Hudi数据源使用基于RDD的HoodieWriteClient API 真正执行对存储的写入。
这些配置可对文件大小、压缩（compression）、并行度、压缩（compaction）、写入模式、清理等底层方面进行完全控制。
尽管Hudi提供了合理的默认设置，但在不同情形下，可能需要对这些配置进行调整以针对特定的工作负载进行优化。 RecordPayload 配置 :
这是Hudi提供的最底层的定制。 RecordPayload定义了如何根据传入的新记录和存储的旧记录来产生新值以进行插入更新。
Hudi提供了诸如OverwriteWithLatestAvroPayload的
默认实现，该实现仅使用最新或最后写入的记录来更新存储。 在数据源和Wr [...]
+ "excerpt":"该页面介绍了几种配置写入或读取Hudi数据集的作业的方法。 简而言之，您可以在几个级别上控制行为。
Spark数据源配置 : 这些配置控制Hudi Spark数据源，提供如下功能： 定义键和分区、选择写操作、指定如何合并记录或选择要读取的视图类型。
WriteClient 配置 : 在内部，Hudi数据源使用基于RDD的HoodieWriteClient API 真正执行对存储的写入。
这些配置可对文件大小、压缩（compression）、并行度、压缩（compaction）、写入模式、清理等底层方面进行完全控制。
尽管Hudi提供了合理的默认设置，但在不同情形下，可能需要对这些配置进行调整以针对特定的工作负载进行优化。 RecordPayload 配置 :
这是Hudi提供的最底层的定制。 RecordPayload定义了如何根据传入的新记录和存储的旧记录来产生新值以进行插入更新。
Hudi提供了诸如OverwriteWithLatestAvroPayload的
默认实现，该实现仅使用最新或最后写入的记录来更新存储。 在数据源和Wr [...]
"tags": [],
"url": "https://hudi.apache.org/cn/docs/configurations.html";,
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
diff --git a/content/docs/oss_hoodie.html b/content/cn/docs/azure_hoodie.html
similarity index 52%
copy from content/docs/oss_hoodie.html
copy to content/cn/docs/azure_hoodie.html
index 411e991..f55b93c 100644
--- a/content/docs/oss_hoodie.html
+++ b/content/cn/docs/azure_hoodie.html
@@ -3,23 +3,23 @@
<head>
<meta charset="utf-8">
-<!-- begin _includes/seo.html --><title>OSS Filesystem - Apache Hudi</title>
-<meta name="description" content="In this page, we explain how to get your
Hudi spark job to store into Aliyun OSS.">
+<!-- begin _includes/seo.html --><title>Azure Filesystem - Apache Hudi</title>
+<meta name="description" content="In this page, we explain how to use Hudi on
Microsoft Azure.">
<meta property="og:type" content="article">
<meta property="og:locale" content="en_US">
<meta property="og:site_name" content="">
-<meta property="og:title" content="OSS Filesystem">
-<meta property="og:url" content="https://hudi.apache.org/docs/oss_hoodie.html";>
+<meta property="og:title" content="Azure Filesystem">
+<meta property="og:url"
content="https://hudi.apache.org/cn/docs/azure_hoodie.html";>
- <meta property="og:description" content="In this page, we explain how to get
your Hudi spark job to store into Aliyun OSS.">
+ <meta property="og:description" content="In this page, we explain how to use
Hudi on Microsoft Azure.">
- <meta property="article:modified_time" content="2020-04-21T17:38:24-04:00">
+ <meta property="article:modified_time" content="2020-05-25T19:00:57-04:00">
@@ -83,15 +83,15 @@
</a>
<ul class="visible-links"><li class="masthead__menu-item">
- <a href="/docs/quick-start-guide.html" target="_self"
>Documentation</a>
+ <a href="/cn/docs/quick-start-guide.html" target="_self" >文档</a>
</li><li class="masthead__menu-item">
- <a href="/community.html" target="_self" >Community</a>
+ <a href="/cn/community.html" target="_self" >社区</a>
</li><li class="masthead__menu-item">
- <a href="/blog.html" target="_self" >Blog</a>
+ <a href="/cn/activity.html" target="_self" >动态</a>
</li><li class="masthead__menu-item">
<a href="https://cwiki.apache.org/confluence/display/HUDI/FAQ";
target="_blank" >FAQ</a>
</li><li class="masthead__menu-item">
- <a href="/releases.html" target="_self" >Releases</a>
+ <a href="/cn/releases.html" target="_self" >发布</a>
</li></ul>
<button class="greedy-nav__toggle hidden" type="button">
<span class="visually-hidden">Toggle menu</span>
@@ -129,12 +129,12 @@
<nav class="nav__list">
<input id="ac-toc" name="accordion-toc" type="checkbox" />
- <label for="ac-toc">Toggle Menu</label>
+ <label for="ac-toc">文档菜单</label>
<ul class="nav__items">
<li>
- <span class="nav__sub-title">Getting Started</span>
+ <span class="nav__sub-title">入门指南</span>
@@ -147,7 +147,7 @@
- <li><a href="/docs/quick-start-guide.html" class="">Quick
Start</a></li>
+ <li><a href="/cn/docs/quick-start-guide.html"
class="">快速开始</a></li>
@@ -158,7 +158,7 @@
- <li><a href="/docs/use_cases.html" class="">Use Cases</a></li>
+ <li><a href="/cn/docs/use_cases.html" class="">使用案例</a></li>
@@ -169,7 +169,7 @@
- <li><a href="/docs/powered_by.html" class="">Talks & Powered
By</a></li>
+ <li><a href="/cn/docs/powered_by.html" class="">演讲 & hudi
用户</a></li>
@@ -180,7 +180,7 @@
- <li><a href="/docs/comparison.html" class="">Comparison</a></li>
+ <li><a href="/cn/docs/comparison.html" class="">对比</a></li>
@@ -191,7 +191,7 @@
- <li><a href="/docs/docker_demo.html" class="">Docker
Demo</a></li>
+ <li><a href="/cn/docs/docker_demo.html" class="">Docker
示例</a></li>
@@ -201,7 +201,7 @@
<li>
- <span class="nav__sub-title">Documentation</span>
+ <span class="nav__sub-title">帮助文档</span>
@@ -214,7 +214,7 @@
- <li><a href="/docs/concepts.html" class="">Concepts</a></li>
+ <li><a href="/cn/docs/concepts.html" class="">概念</a></li>
@@ -225,7 +225,7 @@
- <li><a href="/docs/writing_data.html" class="">Writing
Data</a></li>
+ <li><a href="/cn/docs/writing_data.html" class="">写入数据</a></li>
@@ -236,7 +236,7 @@
- <li><a href="/docs/querying_data.html" class="">Querying
Data</a></li>
+ <li><a href="/cn/docs/querying_data.html" class="">查询数据</a></li>
@@ -247,7 +247,7 @@
- <li><a href="/docs/configurations.html"
class="">Configuration</a></li>
+ <li><a href="/cn/docs/configurations.html" class="">配置</a></li>
@@ -258,7 +258,7 @@
- <li><a href="/docs/performance.html"
class="">Performance</a></li>
+ <li><a href="/cn/docs/performance.html" class="">性能</a></li>
@@ -269,7 +269,7 @@
- <li><a href="/docs/deployment.html" class="">Deployment</a></li>
+ <li><a href="/cn/docs/deployment.html" class="">管理</a></li>
@@ -279,7 +279,7 @@
<li>
- <span class="nav__sub-title">INFO</span>
+ <span class="nav__sub-title">其他信息</span>
@@ -292,7 +292,7 @@
- <li><a href="/docs/docs-versions.html" class="">Docs
Versions</a></li>
+ <li><a href="/cn/docs/docs-versions.html" class="">文档版本</a></li>
@@ -303,7 +303,7 @@
- <li><a href="/docs/privacy.html" class="">Privacy Policy</a></li>
+ <li><a href="/cn/docs/privacy.html" class="">版权信息</a></li>
@@ -326,7 +326,7 @@
<div class="page__inner-wrap">
<header>
- <h1 id="page-title" class="page__title" itemprop="headline">OSS
Filesystem
+ <h1 id="page-title" class="page__title" itemprop="headline">Azure
Filesystem
</h1>
<!-- Output author details if some exist. -->
@@ -341,70 +341,55 @@
}
</style>
- <p>In this page, we explain how to get your Hudi spark job to store
into Aliyun OSS.</p>
+ <p>In this page, we explain how to use Hudi on Microsoft Azure.</p>
-<h2 id="aliyun-oss-configs">Aliyun OSS configs</h2>
+<h2 id="disclaimer">Disclaimer</h2>
-<p>There are two configurations required for Hudi-OSS compatibility:</p>
+<p>This page is maintained by the Hudi community.
+If the information is inaccurate or you have additional information to add.
+Please feel free to create a JIRA ticket. Contribution is highly
appreciated.</p>
+
+<h2 id="supported-storage-system">Supported Storage System</h2>
+
+<p>There are two storage systems support Hudi .</p>
<ul>
- <li>Adding Aliyun OSS Credentials for Hudi</li>
- <li>Adding required Jars to classpath</li>
+ <li>Azure Blob Storage</li>
+ <li>Azure Data Lake Gen 2</li>
</ul>
-<h3 id="aliyun-oss-credentials">Aliyun OSS Credentials</h3>
-
-<p>Add the required configs in your core-site.xml from where Hudi can fetch
them. Replace the <code class="highlighter-rouge">fs.defaultFS</code> with your
OSS bucket name, replace <code class="highlighter-rouge">fs.oss.endpoint</code>
with your OSS endpoint, replace <code
class="highlighter-rouge">fs.oss.accessKeyId</code> with your OSS key, replace
<code class="highlighter-rouge">fs.oss.accessKeySecret</code> with your OSS
secret. Hudi should be able to read/write from the bucket.</p>
-
-<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code> <span class="nt"><property></span>
- <span class="nt"><name></span>fs.defaultFS<span
class="nt"></name></span>
- <span class="nt"><value></span>oss://bucketname/<span
class="nt"></value></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.endpoint<span
class="nt"></name></span>
- <span class="nt"><value></span>oss-endpoint-address<span
class="nt"></value></span>
- <span class="nt"><description></span>Aliyun OSS endpoint to
connect to.<span class="nt"></description></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.accessKeyId<span
class="nt"></name></span>
- <span class="nt"><value></span>oss_key<span
class="nt"></value></span>
- <span class="nt"><description></span>Aliyun access key ID<span
class="nt"></description></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.accessKeySecret<span
class="nt"></name></span>
- <span class="nt"><value></span>oss-secret<span
class="nt"></value></span>
- <span class="nt"><description></span>Aliyun access key secret<span
class="nt"></description></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.impl<span
class="nt"></name></span>
- <span
class="nt"><value></span>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem<span
class="nt"></value></span>
- <span class="nt"></property></span>
-</code></pre></div></div>
-
-<h3 id="aliyun-oss-libs">Aliyun OSS Libs</h3>
-
-<p>Aliyun hadoop libraries jars to add to our pom.xml. Since hadoop-aliyun
depends on the version of hadoop 2.9.1+, you need to use the version of hadoop
2.9.1 or later.</p>
-
-<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nt"><dependency></span>
- <span class="nt"><groupId></span>org.apache.hadoop<span
class="nt"></groupId></span>
- <span class="nt"><artifactId></span>hadoop-aliyun<span
class="nt"></artifactId></span>
- <span class="nt"><version></span>3.2.1<span
class="nt"></version></span>
-<span class="nt"></dependency></span>
-<span class="nt"><dependency></span>
- <span class="nt"><groupId></span>com.aliyun.oss<span
class="nt"></groupId></span>
- <span class="nt"><artifactId></span>aliyun-sdk-oss<span
class="nt"></artifactId></span>
- <span class="nt"><version></span>3.8.1<span
class="nt"></version></span>
-<span class="nt"></dependency></span>
-<span class="nt"><dependency></span>
- <span class="nt"><groupId></span>org.jdom<span
class="nt"></groupId></span>
- <span class="nt"><artifactId></span>jdom<span
class="nt"></artifactId></span>
- <span class="nt"><version></span>1.1<span
class="nt"></version></span>
-<span class="nt"></dependency></span>
-</code></pre></div></div>
+<h2 id="verified-combination-of-spark-and-storage-system">Verified Combination
of Spark and storage system</h2>
+
+<h4 id="hdinsight-spark24-on-azure-data-lake-storage-gen-2">HDInsight Spark2.4
on Azure Data Lake Storage Gen 2</h4>
+<p>This combination works out of the box. No extra config needed.</p>
+
+<h4 id="databricks-spark24-on-azure-data-lake-storage-gen-2">Databricks
Spark2.4 on Azure Data Lake Storage Gen 2</h4>
+<ul>
+ <li>
+ <p>Import Hudi jar to databricks workspace</p>
+ </li>
+ <li>Mount the file system to dbutils.
+ <div class="language-scala highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nv">dbutils</span><span
class="o">.</span><span class="py">fs</span><span class="o">.</span><span
class="py">mount</span><span class="o">(</span>
+ <span class="n">source</span> <span class="k">=</span> <span
class="s">"abfss://[email protected]"</span><span class="o">,</span>
+ <span class="n">mountPoint</span> <span class="k">=</span> <span
class="s">"/mountpoint"</span><span class="o">,</span>
+ <span class="n">extraConfigs</span> <span class="k">=</span> <span
class="n">configs</span><span class="o">)</span>
+</code></pre></div> </div>
+ </li>
+ <li>When writing Hudi dataset, use abfss URL
+ <div class="language-scala highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nv">inputDF</span><span
class="o">.</span><span class="py">write</span>
+ <span class="o">.</span><span class="py">format</span><span
class="o">(</span><span class="s">"org.apache.hudi"</span><span
class="o">)</span>
+ <span class="o">.</span><span class="py">options</span><span
class="o">(</span><span class="n">opts</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">mode</span><span
class="o">(</span><span class="nv">SaveMode</span><span class="o">.</span><span
class="py">Append</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">save</span><span
class="o">(</span><span
class="s">"abfss://<<storage-account>>.dfs.core.windows.net/hudi-tables/customer"</span><span
class="o">)</span>
+</code></pre></div> </div>
+ </li>
+ <li>When reading Hudi dataset, use the mounting point
+ <div class="language-scala highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nv">spark</span><span
class="o">.</span><span class="py">read</span>
+ <span class="o">.</span><span class="py">format</span><span
class="o">(</span><span class="s">"org.apache.hudi"</span><span
class="o">)</span>
+ <span class="o">.</span><span class="py">load</span><span
class="o">(</span><span
class="s">"/mountpoint/hudi-tables/customer"</span><span class="o">)</span>
+</code></pre></div> </div>
+ </li>
+</ul>
</section>
diff --git a/content/cn/docs/configurations.html
b/content/cn/docs/configurations.html
index dd52de2..07a5797 100644
--- a/content/cn/docs/configurations.html
+++ b/content/cn/docs/configurations.html
@@ -383,6 +383,10 @@
S3和Hudi协同工作所需的配置。</li>
<li><a href="/cn/docs/gcs_hoodie.html">Google Cloud Storage</a> <br />
GCS和Hudi协同工作所需的配置。</li>
+ <li><a href="/cn/docs/oss_hoodie.html">Alibaba Cloud OSS</a> <br />
+阿里云和Hudi协同工作所需的配置。</li>
+ <li><a href="/cn/docs/azure_hoodie.html">Microsoft Azure</a> <br />
+Azure和Hudi协同工作所需的配置。</li>
</ul>
<h2 id="spark-datasource">Spark数据源配置</h2>
diff --git a/content/docs/oss_hoodie.html b/content/cn/docs/oss_hoodie.html
similarity index 76%
copy from content/docs/oss_hoodie.html
copy to content/cn/docs/oss_hoodie.html
index 411e991..b1a754b 100644
--- a/content/docs/oss_hoodie.html
+++ b/content/cn/docs/oss_hoodie.html
@@ -4,22 +4,22 @@
<meta charset="utf-8">
<!-- begin _includes/seo.html --><title>OSS Filesystem - Apache Hudi</title>
-<meta name="description" content="In this page, we explain how to get your
Hudi spark job to store into Aliyun OSS.">
+<meta name="description" content="这个页面描述了如何让你的Hudi spark任务使用Aliyun OSS存储。">
<meta property="og:type" content="article">
<meta property="og:locale" content="en_US">
<meta property="og:site_name" content="">
<meta property="og:title" content="OSS Filesystem">
-<meta property="og:url" content="https://hudi.apache.org/docs/oss_hoodie.html";>
+<meta property="og:url"
content="https://hudi.apache.org/cn/docs/oss_hoodie.html";>
- <meta property="og:description" content="In this page, we explain how to get
your Hudi spark job to store into Aliyun OSS.">
+ <meta property="og:description" content="这个页面描述了如何让你的Hudi spark任务使用Aliyun
OSS存储。">
- <meta property="article:modified_time" content="2020-04-21T17:38:24-04:00">
+ <meta property="article:modified_time" content="2020-04-21T18:50:50-04:00">
@@ -83,15 +83,15 @@
</a>
<ul class="visible-links"><li class="masthead__menu-item">
- <a href="/docs/quick-start-guide.html" target="_self"
>Documentation</a>
+ <a href="/cn/docs/quick-start-guide.html" target="_self" >文档</a>
</li><li class="masthead__menu-item">
- <a href="/community.html" target="_self" >Community</a>
+ <a href="/cn/community.html" target="_self" >社区</a>
</li><li class="masthead__menu-item">
- <a href="/blog.html" target="_self" >Blog</a>
+ <a href="/cn/activity.html" target="_self" >动态</a>
</li><li class="masthead__menu-item">
<a href="https://cwiki.apache.org/confluence/display/HUDI/FAQ";
target="_blank" >FAQ</a>
</li><li class="masthead__menu-item">
- <a href="/releases.html" target="_self" >Releases</a>
+ <a href="/cn/releases.html" target="_self" >发布</a>
</li></ul>
<button class="greedy-nav__toggle hidden" type="button">
<span class="visually-hidden">Toggle menu</span>
@@ -129,12 +129,12 @@
<nav class="nav__list">
<input id="ac-toc" name="accordion-toc" type="checkbox" />
- <label for="ac-toc">Toggle Menu</label>
+ <label for="ac-toc">文档菜单</label>
<ul class="nav__items">
<li>
- <span class="nav__sub-title">Getting Started</span>
+ <span class="nav__sub-title">入门指南</span>
@@ -147,7 +147,7 @@
- <li><a href="/docs/quick-start-guide.html" class="">Quick
Start</a></li>
+ <li><a href="/cn/docs/quick-start-guide.html"
class="">快速开始</a></li>
@@ -158,7 +158,7 @@
- <li><a href="/docs/use_cases.html" class="">Use Cases</a></li>
+ <li><a href="/cn/docs/use_cases.html" class="">使用案例</a></li>
@@ -169,7 +169,7 @@
- <li><a href="/docs/powered_by.html" class="">Talks & Powered
By</a></li>
+ <li><a href="/cn/docs/powered_by.html" class="">演讲 & hudi
用户</a></li>
@@ -180,7 +180,7 @@
- <li><a href="/docs/comparison.html" class="">Comparison</a></li>
+ <li><a href="/cn/docs/comparison.html" class="">对比</a></li>
@@ -191,7 +191,7 @@
- <li><a href="/docs/docker_demo.html" class="">Docker
Demo</a></li>
+ <li><a href="/cn/docs/docker_demo.html" class="">Docker
示例</a></li>
@@ -201,7 +201,7 @@
<li>
- <span class="nav__sub-title">Documentation</span>
+ <span class="nav__sub-title">帮助文档</span>
@@ -214,7 +214,7 @@
- <li><a href="/docs/concepts.html" class="">Concepts</a></li>
+ <li><a href="/cn/docs/concepts.html" class="">概念</a></li>
@@ -225,7 +225,7 @@
- <li><a href="/docs/writing_data.html" class="">Writing
Data</a></li>
+ <li><a href="/cn/docs/writing_data.html" class="">写入数据</a></li>
@@ -236,7 +236,7 @@
- <li><a href="/docs/querying_data.html" class="">Querying
Data</a></li>
+ <li><a href="/cn/docs/querying_data.html" class="">查询数据</a></li>
@@ -247,7 +247,7 @@
- <li><a href="/docs/configurations.html"
class="">Configuration</a></li>
+ <li><a href="/cn/docs/configurations.html" class="">配置</a></li>
@@ -258,7 +258,7 @@
- <li><a href="/docs/performance.html"
class="">Performance</a></li>
+ <li><a href="/cn/docs/performance.html" class="">性能</a></li>
@@ -269,7 +269,7 @@
- <li><a href="/docs/deployment.html" class="">Deployment</a></li>
+ <li><a href="/cn/docs/deployment.html" class="">管理</a></li>
@@ -279,7 +279,7 @@
<li>
- <span class="nav__sub-title">INFO</span>
+ <span class="nav__sub-title">其他信息</span>
@@ -292,7 +292,7 @@
- <li><a href="/docs/docs-versions.html" class="">Docs
Versions</a></li>
+ <li><a href="/cn/docs/docs-versions.html" class="">文档版本</a></li>
@@ -303,7 +303,7 @@
- <li><a href="/docs/privacy.html" class="">Privacy Policy</a></li>
+ <li><a href="/cn/docs/privacy.html" class="">版权信息</a></li>
@@ -341,20 +341,20 @@
}
</style>
- <p>In this page, we explain how to get your Hudi spark job to store
into Aliyun OSS.</p>
+ <p>这个页面描述了如何让你的Hudi spark任务使用Aliyun OSS存储。</p>
-<h2 id="aliyun-oss-configs">Aliyun OSS configs</h2>
+<h2 id="aliyun-oss-部署">Aliyun OSS 部署</h2>
-<p>There are two configurations required for Hudi-OSS compatibility:</p>
+<p>为了让Hudi使用OSS，需要增加两部分的配置:</p>
<ul>
- <li>Adding Aliyun OSS Credentials for Hudi</li>
- <li>Adding required Jars to classpath</li>
+ <li>为Hidi增加Aliyun OSS的相关配置</li>
+ <li>增加Jar包的MVN依赖</li>
</ul>
-<h3 id="aliyun-oss-credentials">Aliyun OSS Credentials</h3>
+<h3 id="aliyun-oss-相关的配置">Aliyun OSS 相关的配置</h3>
-<p>Add the required configs in your core-site.xml from where Hudi can fetch
them. Replace the <code class="highlighter-rouge">fs.defaultFS</code> with your
OSS bucket name, replace <code class="highlighter-rouge">fs.oss.endpoint</code>
with your OSS endpoint, replace <code
class="highlighter-rouge">fs.oss.accessKeyId</code> with your OSS key, replace
<code class="highlighter-rouge">fs.oss.accessKeySecret</code> with your OSS
secret. Hudi should be able to read/write from the bucket.</p>
+<p>新增下面的配置到你的Hudi能访问的core-site.xml文件。使用你的OSS bucket name替换掉<code
class="highlighter-rouge">fs.defaultFS</code>，使用OSS endpoint地址替换<code
class="highlighter-rouge">fs.oss.endpoint</code>，使用OSS的key和secret分别替换<code
class="highlighter-rouge">fs.oss.accessKeyId</code>和<code
class="highlighter-rouge">fs.oss.accessKeySecret</code>。主要Hudi就能读写相应的bucket。</p>
<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code> <span class="nt"><property></span>
<span class="nt"><name></span>fs.defaultFS<span
class="nt"></name></span>
@@ -387,7 +387,7 @@
<h3 id="aliyun-oss-libs">Aliyun OSS Libs</h3>
-<p>Aliyun hadoop libraries jars to add to our pom.xml. Since hadoop-aliyun
depends on the version of hadoop 2.9.1+, you need to use the version of hadoop
2.9.1 or later.</p>
+<p>新增Aliyun hadoop的jar包的MVN依赖到pom.xml文件。由于hadoop-aliyun依赖hadoop
2.9.1+，因此你需要使用hadoop 2.9.1或更新的版本。</p>
<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nt"><dependency></span>
<span class="nt"><groupId></span>org.apache.hadoop<span
class="nt"></groupId></span>
diff --git a/content/docs/oss_hoodie.html b/content/docs/azure_hoodie.html
similarity index 64%
copy from content/docs/oss_hoodie.html
copy to content/docs/azure_hoodie.html
index 411e991..312d1ea 100644
--- a/content/docs/oss_hoodie.html
+++ b/content/docs/azure_hoodie.html
@@ -3,23 +3,23 @@
<head>
<meta charset="utf-8">
-<!-- begin _includes/seo.html --><title>OSS Filesystem - Apache Hudi</title>
-<meta name="description" content="In this page, we explain how to get your
Hudi spark job to store into Aliyun OSS.">
+<!-- begin _includes/seo.html --><title>Azure Filesystem - Apache Hudi</title>
+<meta name="description" content="In this page, we explain how to use Hudi on
Microsoft Azure.">
<meta property="og:type" content="article">
<meta property="og:locale" content="en_US">
<meta property="og:site_name" content="">
-<meta property="og:title" content="OSS Filesystem">
-<meta property="og:url" content="https://hudi.apache.org/docs/oss_hoodie.html";>
+<meta property="og:title" content="Azure Filesystem">
+<meta property="og:url"
content="https://hudi.apache.org/docs/azure_hoodie.html";>
- <meta property="og:description" content="In this page, we explain how to get
your Hudi spark job to store into Aliyun OSS.">
+ <meta property="og:description" content="In this page, we explain how to use
Hudi on Microsoft Azure.">
- <meta property="article:modified_time" content="2020-04-21T17:38:24-04:00">
+ <meta property="article:modified_time" content="2020-05-25T19:00:57-04:00">
@@ -326,7 +326,7 @@
<div class="page__inner-wrap">
<header>
- <h1 id="page-title" class="page__title" itemprop="headline">OSS
Filesystem
+ <h1 id="page-title" class="page__title" itemprop="headline">Azure
Filesystem
</h1>
<!-- Output author details if some exist. -->
@@ -341,70 +341,55 @@
}
</style>
- <p>In this page, we explain how to get your Hudi spark job to store
into Aliyun OSS.</p>
+ <p>In this page, we explain how to use Hudi on Microsoft Azure.</p>
-<h2 id="aliyun-oss-configs">Aliyun OSS configs</h2>
+<h2 id="disclaimer">Disclaimer</h2>
-<p>There are two configurations required for Hudi-OSS compatibility:</p>
+<p>This page is maintained by the Hudi community.
+If the information is inaccurate or you have additional information to add.
+Please feel free to create a JIRA ticket. Contribution is highly
appreciated.</p>
+
+<h2 id="supported-storage-system">Supported Storage System</h2>
+
+<p>There are two storage systems support Hudi .</p>
<ul>
- <li>Adding Aliyun OSS Credentials for Hudi</li>
- <li>Adding required Jars to classpath</li>
+ <li>Azure Blob Storage</li>
+ <li>Azure Data Lake Gen 2</li>
</ul>
-<h3 id="aliyun-oss-credentials">Aliyun OSS Credentials</h3>
-
-<p>Add the required configs in your core-site.xml from where Hudi can fetch
them. Replace the <code class="highlighter-rouge">fs.defaultFS</code> with your
OSS bucket name, replace <code class="highlighter-rouge">fs.oss.endpoint</code>
with your OSS endpoint, replace <code
class="highlighter-rouge">fs.oss.accessKeyId</code> with your OSS key, replace
<code class="highlighter-rouge">fs.oss.accessKeySecret</code> with your OSS
secret. Hudi should be able to read/write from the bucket.</p>
-
-<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code> <span class="nt"><property></span>
- <span class="nt"><name></span>fs.defaultFS<span
class="nt"></name></span>
- <span class="nt"><value></span>oss://bucketname/<span
class="nt"></value></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.endpoint<span
class="nt"></name></span>
- <span class="nt"><value></span>oss-endpoint-address<span
class="nt"></value></span>
- <span class="nt"><description></span>Aliyun OSS endpoint to
connect to.<span class="nt"></description></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.accessKeyId<span
class="nt"></name></span>
- <span class="nt"><value></span>oss_key<span
class="nt"></value></span>
- <span class="nt"><description></span>Aliyun access key ID<span
class="nt"></description></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.accessKeySecret<span
class="nt"></name></span>
- <span class="nt"><value></span>oss-secret<span
class="nt"></value></span>
- <span class="nt"><description></span>Aliyun access key secret<span
class="nt"></description></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.impl<span
class="nt"></name></span>
- <span
class="nt"><value></span>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem<span
class="nt"></value></span>
- <span class="nt"></property></span>
-</code></pre></div></div>
-
-<h3 id="aliyun-oss-libs">Aliyun OSS Libs</h3>
-
-<p>Aliyun hadoop libraries jars to add to our pom.xml. Since hadoop-aliyun
depends on the version of hadoop 2.9.1+, you need to use the version of hadoop
2.9.1 or later.</p>
-
-<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nt"><dependency></span>
- <span class="nt"><groupId></span>org.apache.hadoop<span
class="nt"></groupId></span>
- <span class="nt"><artifactId></span>hadoop-aliyun<span
class="nt"></artifactId></span>
- <span class="nt"><version></span>3.2.1<span
class="nt"></version></span>
-<span class="nt"></dependency></span>
-<span class="nt"><dependency></span>
- <span class="nt"><groupId></span>com.aliyun.oss<span
class="nt"></groupId></span>
- <span class="nt"><artifactId></span>aliyun-sdk-oss<span
class="nt"></artifactId></span>
- <span class="nt"><version></span>3.8.1<span
class="nt"></version></span>
-<span class="nt"></dependency></span>
-<span class="nt"><dependency></span>
- <span class="nt"><groupId></span>org.jdom<span
class="nt"></groupId></span>
- <span class="nt"><artifactId></span>jdom<span
class="nt"></artifactId></span>
- <span class="nt"><version></span>1.1<span
class="nt"></version></span>
-<span class="nt"></dependency></span>
-</code></pre></div></div>
+<h2 id="verified-combination-of-spark-and-storage-system">Verified Combination
of Spark and storage system</h2>
+
+<h4 id="hdinsight-spark24-on-azure-data-lake-storage-gen-2">HDInsight Spark2.4
on Azure Data Lake Storage Gen 2</h4>
+<p>This combination works out of the box. No extra config needed.</p>
+
+<h4 id="databricks-spark24-on-azure-data-lake-storage-gen-2">Databricks
Spark2.4 on Azure Data Lake Storage Gen 2</h4>
+<ul>
+ <li>
+ <p>Import Hudi jar to databricks workspace</p>
+ </li>
+ <li>Mount the file system to dbutils.
+ <div class="language-scala highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nv">dbutils</span><span
class="o">.</span><span class="py">fs</span><span class="o">.</span><span
class="py">mount</span><span class="o">(</span>
+ <span class="n">source</span> <span class="k">=</span> <span
class="s">"abfss://[email protected]"</span><span class="o">,</span>
+ <span class="n">mountPoint</span> <span class="k">=</span> <span
class="s">"/mountpoint"</span><span class="o">,</span>
+ <span class="n">extraConfigs</span> <span class="k">=</span> <span
class="n">configs</span><span class="o">)</span>
+</code></pre></div> </div>
+ </li>
+ <li>When writing Hudi dataset, use abfss URL
+ <div class="language-scala highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nv">inputDF</span><span
class="o">.</span><span class="py">write</span>
+ <span class="o">.</span><span class="py">format</span><span
class="o">(</span><span class="s">"org.apache.hudi"</span><span
class="o">)</span>
+ <span class="o">.</span><span class="py">options</span><span
class="o">(</span><span class="n">opts</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">mode</span><span
class="o">(</span><span class="nv">SaveMode</span><span class="o">.</span><span
class="py">Append</span><span class="o">)</span>
+ <span class="o">.</span><span class="py">save</span><span
class="o">(</span><span
class="s">"abfss://<<storage-account>>.dfs.core.windows.net/hudi-tables/customer"</span><span
class="o">)</span>
+</code></pre></div> </div>
+ </li>
+ <li>When reading Hudi dataset, use the mounting point
+ <div class="language-scala highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nv">spark</span><span
class="o">.</span><span class="py">read</span>
+ <span class="o">.</span><span class="py">format</span><span
class="o">(</span><span class="s">"org.apache.hudi"</span><span
class="o">)</span>
+ <span class="o">.</span><span class="py">load</span><span
class="o">(</span><span
class="s">"/mountpoint/hudi-tables/customer"</span><span class="o">)</span>
+</code></pre></div> </div>
+ </li>
+</ul>
</section>
diff --git a/content/docs/configurations.html b/content/docs/configurations.html
index dd99967..a09d4c6 100644
--- a/content/docs/configurations.html
+++ b/content/docs/configurations.html
@@ -381,6 +381,10 @@ to cloud stores.</p>
Configurations required for S3 and Hudi co-operability.</li>
<li><a href="/docs/gcs_hoodie">Google Cloud Storage</a> <br />
Configurations required for GCS and Hudi co-operability.</li>
+ <li><a href="/docs/oss_hoodie.html">Alibaba Cloud OSS</a> <br />
+Configurations required for OSS and Hudi co-operability.</li>
+ <li><a href="/docs/azure_hoodie.html">Microsoft Azure</a> <br />
+Configurations required for Azure and Hudi co-operability.</li>
</ul>
<h2 id="spark-datasource">Spark Datasource Configs</h2>
diff --git a/content/docs/oss_hoodie.html b/content/docs/oss_hoodie.html
index 411e991..dfc6e7e 100644
--- a/content/docs/oss_hoodie.html
+++ b/content/docs/oss_hoodie.html
@@ -356,33 +356,33 @@
<p>Add the required configs in your core-site.xml from where Hudi can fetch
them. Replace the <code class="highlighter-rouge">fs.defaultFS</code> with your
OSS bucket name, replace <code class="highlighter-rouge">fs.oss.endpoint</code>
with your OSS endpoint, replace <code
class="highlighter-rouge">fs.oss.accessKeyId</code> with your OSS key, replace
<code class="highlighter-rouge">fs.oss.accessKeySecret</code> with your OSS
secret. Hudi should be able to read/write from the bucket.</p>
-<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code> <span class="nt"><property></span>
- <span class="nt"><name></span>fs.defaultFS<span
class="nt"></name></span>
- <span class="nt"><value></span>oss://bucketname/<span
class="nt"></value></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.endpoint<span
class="nt"></name></span>
- <span class="nt"><value></span>oss-endpoint-address<span
class="nt"></value></span>
- <span class="nt"><description></span>Aliyun OSS endpoint to
connect to.<span class="nt"></description></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.accessKeyId<span
class="nt"></name></span>
- <span class="nt"><value></span>oss_key<span
class="nt"></value></span>
- <span class="nt"><description></span>Aliyun access key ID<span
class="nt"></description></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.accessKeySecret<span
class="nt"></name></span>
- <span class="nt"><value></span>oss-secret<span
class="nt"></value></span>
- <span class="nt"><description></span>Aliyun access key secret<span
class="nt"></description></span>
- <span class="nt"></property></span>
-
- <span class="nt"><property></span>
- <span class="nt"><name></span>fs.oss.impl<span
class="nt"></name></span>
- <span
class="nt"><value></span>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem<span
class="nt"></value></span>
- <span class="nt"></property></span>
+<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nt"><property></span>
+ <span class="nt"><name></span>fs.defaultFS<span
class="nt"></name></span>
+ <span class="nt"><value></span>oss://bucketname/<span
class="nt"></value></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>fs.oss.endpoint<span
class="nt"></name></span>
+ <span class="nt"><value></span>oss-endpoint-address<span
class="nt"></value></span>
+ <span class="nt"><description></span>Aliyun OSS endpoint to connect
to.<span class="nt"></description></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>fs.oss.accessKeyId<span
class="nt"></name></span>
+ <span class="nt"><value></span>oss_key<span
class="nt"></value></span>
+ <span class="nt"><description></span>Aliyun access key ID<span
class="nt"></description></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>fs.oss.accessKeySecret<span
class="nt"></name></span>
+ <span class="nt"><value></span>oss-secret<span
class="nt"></value></span>
+ <span class="nt"><description></span>Aliyun access key secret<span
class="nt"></description></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>fs.oss.impl<span
class="nt"></name></span>
+ <span
class="nt"><value></span>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem<span
class="nt"></value></span>
+<span class="nt"></property></span>
</code></pre></div></div>
<h3 id="aliyun-oss-libs">Aliyun OSS Libs</h3>
@@ -390,19 +390,19 @@
<p>Aliyun hadoop libraries jars to add to our pom.xml. Since hadoop-aliyun
depends on the version of hadoop 2.9.1+, you need to use the version of hadoop
2.9.1 or later.</p>
<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="nt"><dependency></span>
- <span class="nt"><groupId></span>org.apache.hadoop<span
class="nt"></groupId></span>
- <span class="nt"><artifactId></span>hadoop-aliyun<span
class="nt"></artifactId></span>
- <span class="nt"><version></span>3.2.1<span
class="nt"></version></span>
+ <span class="nt"><groupId></span>org.apache.hadoop<span
class="nt"></groupId></span>
+ <span class="nt"><artifactId></span>hadoop-aliyun<span
class="nt"></artifactId></span>
+ <span class="nt"><version></span>3.2.1<span
class="nt"></version></span>
<span class="nt"></dependency></span>
<span class="nt"><dependency></span>
- <span class="nt"><groupId></span>com.aliyun.oss<span
class="nt"></groupId></span>
- <span class="nt"><artifactId></span>aliyun-sdk-oss<span
class="nt"></artifactId></span>
- <span class="nt"><version></span>3.8.1<span
class="nt"></version></span>
+ <span class="nt"><groupId></span>com.aliyun.oss<span
class="nt"></groupId></span>
+ <span class="nt"><artifactId></span>aliyun-sdk-oss<span
class="nt"></artifactId></span>
+ <span class="nt"><version></span>3.8.1<span
class="nt"></version></span>
<span class="nt"></dependency></span>
<span class="nt"><dependency></span>
- <span class="nt"><groupId></span>org.jdom<span
class="nt"></groupId></span>
- <span class="nt"><artifactId></span>jdom<span
class="nt"></artifactId></span>
- <span class="nt"><version></span>1.1<span
class="nt"></version></span>
+ <span class="nt"><groupId></span>org.jdom<span
class="nt"></groupId></span>
+ <span class="nt"><artifactId></span>jdom<span
class="nt"></artifactId></span>
+ <span class="nt"><version></span>1.1<span
class="nt"></version></span>
<span class="nt"></dependency></span>
</code></pre></div></div>
diff --git a/content/sitemap.xml b/content/sitemap.xml
index ac31c66..3261bf9 100644
--- a/content/sitemap.xml
+++ b/content/sitemap.xml
@@ -429,7 +429,7 @@
<lastmod>2019-12-30T14:59:57-05:00</lastmod>
</url>
<url>
-<loc>https://hudi.apache.org/docs/oss_hoodie.html</loc>
+<loc>https://hudi.apache.org/cn/docs/oss_hoodie.html</loc>
<lastmod>2020-04-21T18:50:50-04:00</lastmod>
</url>
<url>
@@ -437,6 +437,14 @@
<lastmod>2020-04-21T17:38:24-04:00</lastmod>
</url>
<url>
+<loc>https://hudi.apache.org/cn/docs/azure_hoodie.html</loc>
+<lastmod>2020-05-25T19:00:57-04:00</lastmod>
+</url>
+<url>
+<loc>https://hudi.apache.org/docs/azure_hoodie.html</loc>
+<lastmod>2020-05-25T19:00:57-04:00</lastmod>
+</url>
+<url>
<loc>https://hudi.apache.org/cn/docs/quick-start-guide.html</loc>
<lastmod>2019-12-30T14:59:57-05:00</lastmod>
</url>
![https://hudi.apache.org/assets/images/500x300.png"},{](https://hudi.apache.org/assets/images/500x300.png"},{){kind=link}