This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/asf-site by this push:
new bd10ad7 Travis CI build asf-site
bd10ad7 is described below
commit bd10ad7c024baf77bd8f5db27e74d548bf4e10a9
Author: CI <[email protected]>
AuthorDate: Mon Aug 3 12:21:04 2020 +0000
Travis CI build asf-site
---
content/assets/js/lunr/lunr-store.js | 12 ++-
content/cn/docs/cloud.html | 2 +
content/cn/docs/{cloud.html => cos_hoodie.html} | 103 +++++++++++++++++------
content/docs/cloud.html | 2 +
content/docs/{cloud.html => cos_hoodie.html} | 104 ++++++++++++++++++------
content/sitemap.xml | 8 ++
6 files changed, 177 insertions(+), 54 deletions(-)
diff --git a/content/assets/js/lunr/lunr-store.js
b/content/assets/js/lunr/lunr-store.js
index 4314edf..5e4b619 100644
--- a/content/assets/js/lunr/lunr-store.js
+++ b/content/assets/js/lunr/lunr-store.js
@@ -749,6 +749,16 @@ var store = [{
"tags": [],
"url": "https://hudi.apache.org/docs/azure_hoodie.html";,
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
+ "title": "COS Filesystem",
+ "excerpt":"这个页面描述了如何让你的Hudi spark任务使用Tencent Cloud COS存储。 Tencent
Cloud COS 部署 为了让Hudi使用COS，需要增加两部分的配置: 为Hidi增加Tencent Cloud COS的相关配置
增加Jar包的MVN依赖 Tencent Cloud COS 相关的配置 新增下面的配置到你的Hudi能访问的core-site.xml文件。使用你的COS
bucket
name替换掉fs.defaultFS，使用COS的key和secret分别替换fs.cosn.userinfo.secretKey和fs.cosn.userinfo.secretId。主要Hudi就能读写相应的bucket。
<property> <name>fs.defaultFS</name>
<value>cosn://bucketname</value> <description>COS bucket
name</description [...]
+ "tags": [],
+ "url": "https://hudi.apache.org/cn/docs/cos_hoodie.html";,
+ "teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
+ "title": "COS Filesystem",
+ "excerpt":"In this page, we explain how to get your Hudi spark job to
store into Tencent Cloud COS. Tencent Cloud COS configs There are two
configurations required for Hudi-COS compatibility: Adding Tencent Cloud COS
Credentials for Hudi Adding required Jars to classpath Tencent Cloud COS
Credentials Add the required configs...","categories": [],
+ "tags": [],
+ "url": "https://hudi.apache.org/docs/cos_hoodie.html";,
+ "teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
"title": "Quick-Start Guide",
"excerpt":"本指南通过使用spark-shell简要介绍了Hudi功能。使用Spark数据源，我们将通过代码段展示如何插入和更新Hudi的默认存储类型数据集：
写时复制。每次写操作之后，我们还将展示如何读取快照和增量数据。 设置spark-shell
Hudi适用于Spark-2.x版本。您可以按照此处的说明设置spark。 在提取的目录中，使用spark-shell运行Hudi：
bin/spark-shell --packages org.apache.hudi:hudi-spark-bundle:0.5.0-incubating
--conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer'
设置表名、基本路径和数据生成器来为本指南生成记录。 import org.apache.hudi.QuickstartUtils._ import
scala.collection.JavaConversions._ import org.apache.spark.sql.Sa [...]
"tags": [],
@@ -855,7 +865,7 @@ var store = [{
"url": "https://hudi.apache.org/docs/deployment.html";,
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
"title": "云储存",
- "excerpt":"与云存储连接 无论使用RDD/WriteClient API还是数据源，以下信息都有助于配置对云存储的访问。
AWS S3 S3和Hudi协同工作所需的配置。 Google Cloud Storage GCS和Hudi协同工作所需的配置。
Alibaba Cloud OSS 阿里云和Hudi协同工作所需的配置。 Microsoft Azure Azure和Hudi协同工作所需的配置。
","categories": [],
+ "excerpt":"与云存储连接 无论使用RDD/WriteClient API还是数据源，以下信息都有助于配置对云存储的访问。
AWS S3 S3和Hudi协同工作所需的配置。 Google Cloud Storage GCS和Hudi协同工作所需的配置。
Alibaba Cloud OSS 阿里云和Hudi协同工作所需的配置。 Microsoft Azure Azure和Hudi协同工作所需的配置。
Tencent Cloud Object Storage COS和Hudi协同工作所需的配置。 ","categories": [],
"tags": [],
"url": "https://hudi.apache.org/cn/docs/cloud.html";,
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
diff --git a/content/cn/docs/cloud.html b/content/cn/docs/cloud.html
index 9c0288d..d536b73 100644
--- a/content/cn/docs/cloud.html
+++ b/content/cn/docs/cloud.html
@@ -373,6 +373,8 @@ GCS和Hudi协同工作所需的配置。</li>
阿里云和Hudi协同工作所需的配置。</li>
<li><a href="/cn/docs/azure_hoodie.html">Microsoft Azure</a> <br />
Azure和Hudi协同工作所需的配置。</li>
+ <li><a href="/cn/docs/cos_hoodie.html">Tencent Cloud Object Storage</a> <br
/>
+COS和Hudi协同工作所需的配置。</li>
</ul>
</section>
diff --git a/content/cn/docs/cloud.html b/content/cn/docs/cos_hoodie.html
similarity index 63%
copy from content/cn/docs/cloud.html
copy to content/cn/docs/cos_hoodie.html
index 9c0288d..9bb5dc4 100644
--- a/content/cn/docs/cloud.html
+++ b/content/cn/docs/cos_hoodie.html
@@ -3,23 +3,23 @@
<head>
<meta charset="utf-8">
-<!-- begin _includes/seo.html --><title>云储存 - Apache Hudi</title>
-<meta name="description" content="与云存储连接">
+<!-- begin _includes/seo.html --><title>COS Filesystem - Apache Hudi</title>
+<meta name="description" content="这个页面描述了如何让你的Hudi spark任务使用Tencent Cloud
COS存储。">
<meta property="og:type" content="article">
<meta property="og:locale" content="en_US">
<meta property="og:site_name" content="">
-<meta property="og:title" content="云储存">
-<meta property="og:url" content="https://hudi.apache.org/cn/docs/cloud.html";>
+<meta property="og:title" content="COS Filesystem">
+<meta property="og:url"
content="https://hudi.apache.org/cn/docs/cos_hoodie.html";>
- <meta property="og:description" content="与云存储连接">
+ <meta property="og:description" content="这个页面描述了如何让你的Hudi spark任务使用Tencent
Cloud COS存储。">
- <meta property="article:modified_time" content="2019-06-16T21:59:57-04:00">
+ <meta property="article:modified_time" content="2020-04-21T18:50:50-04:00">
@@ -284,7 +284,7 @@
- <li><a href="/cn/docs/cloud.html" class="active">云储存</a></li>
+ <li><a href="/cn/docs/cloud.html" class="">云储存</a></li>
@@ -342,7 +342,7 @@
<div class="page__inner-wrap">
<header>
- <h1 id="page-title" class="page__title" itemprop="headline">云储存
+ <h1 id="page-title" class="page__title" itemprop="headline">COS
Filesystem
</h1>
<!-- Output author details if some exist. -->
@@ -351,28 +351,79 @@
<section class="page__content" itemprop="text">
- <aside class="sidebar__right sticky">
- <nav class="toc">
- <header><h4 class="nav__title"><i class="fas fa-file-alt"></i> IN
THIS PAGE</h4></header>
- <ul class="toc__menu">
- <li><a href="#与云存储连接">与云存储连接</a></li>
-</ul>
- </nav>
- </aside>
+ <style>
+ .page {
+ padding-right: 0 !important;
+ }
+ </style>
- <h2 id="与云存储连接">与云存储连接</h2>
+ <p>这个页面描述了如何让你的Hudi spark任务使用Tencent Cloud COS存储。</p>
+
+<h2 id="tencent-cloud-cos-部署">Tencent Cloud COS 部署</h2>
+
+<p>为了让Hudi使用COS，需要增加两部分的配置:</p>
+
+<ul>
+ <li>为Hidi增加Tencent Cloud COS的相关配置</li>
+ <li>增加Jar包的MVN依赖</li>
+</ul>
-<p>无论使用RDD/WriteClient API还是数据源，以下信息都有助于配置对云存储的访问。</p>
+<h3 id="tencent-cloud-cos-相关的配置">Tencent Cloud COS 相关的配置</h3>
+
+<p>新增下面的配置到你的Hudi能访问的core-site.xml文件。使用你的COS bucket name替换掉<code
class="highlighter-rouge">fs.defaultFS</code>，使用COS的key和secret分别替换<code
class="highlighter-rouge">fs.cosn.userinfo.secretKey</code>和<code
class="highlighter-rouge">fs.cosn.userinfo.secretId</code>。主要Hudi就能读写相应的bucket。</p>
+
+<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code> <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.defaultFS<span
class="nt"></name></span>
+ <span class="nt"><value></span>cosn://bucketname<span
class="nt"></value></span>
+ <span class="nt"><description></span>COS bucket name<span
class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.cosn.userinfo.secretId<span
class="nt"></name></span>
+ <span class="nt"><value></span>cos-secretId<span
class="nt"></value></span>
+ <span class="nt"><description></span>Tencent Cloud Secret
Id<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.cosn.userinfo.secretKey<span
class="nt"></name></span>
+ <span class="nt"><value></span>cos-secretkey<span
class="nt"></value></span>
+ <span class="nt"><description></span>Tencent Cloud Secret
Key<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.cosn.bucket.region<span
class="nt"></name></span>
+ <span class="nt"><value></span>ap-region<span
class="nt"></value></span>
+ <span class="nt"><description></span>The region where the bucket
is located.<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span
class="nt"><name></span>fs.cosn.bucket.endpoint_suffix<span
class="nt"></name></span>
+ <span class="nt"><value></span>cos.endpoint.suffix<span
class="nt"></value></span>
+ <span class="nt"><description></span>
+ COS endpoint to connect to.
+ For public cloud users, it is recommended not to set this option,
and only the correct area field is required.
+ <span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.cosn.impl<span
class="nt"></name></span>
+ <span
class="nt"><value></span>org.apache.hadoop.fs.CosFileSystem<span
class="nt"></value></span>
+ <span class="nt"><description></span>The implementation class of
the CosN Filesystem.<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span
class="nt"><name></span>fs.AbstractFileSystem.cosn.impl<span
class="nt"></name></span>
+ <span class="nt"><value></span>org.apache.hadoop.fs.CosN<span
class="nt"></value></span>
+ <span class="nt"><description></span>The implementation class of
the CosN AbstractFileSystem.<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+</code></pre></div></div>
+
+<h3 id="tencent-cloud-cos-libs">Tencent Cloud COS Libs</h3>
+<p>添加COS依赖jar包到classpath</p>
<ul>
- <li><a href="/cn/docs/s3_hoodie.html">AWS S3</a> <br />
-S3和Hudi协同工作所需的配置。</li>
- <li><a href="/cn/docs/gcs_hoodie.html">Google Cloud Storage</a> <br />
-GCS和Hudi协同工作所需的配置。</li>
- <li><a href="/cn/docs/oss_hoodie.html">Alibaba Cloud OSS</a> <br />
-阿里云和Hudi协同工作所需的配置。</li>
- <li><a href="/cn/docs/azure_hoodie.html">Microsoft Azure</a> <br />
-Azure和Hudi协同工作所需的配置。</li>
+ <li>org.apache.hadoop:hadoop-cos:2.8.5</li>
</ul>
</section>
diff --git a/content/docs/cloud.html b/content/docs/cloud.html
index e8d11d6..7e7e06f 100644
--- a/content/docs/cloud.html
+++ b/content/docs/cloud.html
@@ -385,6 +385,8 @@ Configurations required for GCS and Hudi
co-operability.</li>
Configurations required for OSS and Hudi co-operability.</li>
<li><a href="/docs/azure_hoodie.html">Microsoft Azure</a> <br />
Configurations required for Azure and Hudi co-operability.</li>
+ <li><a href="/docs/cos_hoodie.html">Tencent Cloud Object Storage</a> <br />
+ Configurations required for COS and Hudi co-operability.</li>
</ul>
</section>
diff --git a/content/docs/cloud.html b/content/docs/cos_hoodie.html
similarity index 62%
copy from content/docs/cloud.html
copy to content/docs/cos_hoodie.html
index e8d11d6..fa2438c 100644
--- a/content/docs/cloud.html
+++ b/content/docs/cos_hoodie.html
@@ -3,23 +3,23 @@
<head>
<meta charset="utf-8">
-<!-- begin _includes/seo.html --><title>Cloud Storage - Apache Hudi</title>
-<meta name="description" content="Talking to Cloud Storage">
+<!-- begin _includes/seo.html --><title>COS Filesystem - Apache Hudi</title>
+<meta name="description" content="In this page, we explain how to get your
Hudi spark job to store into Tencent Cloud COS.">
<meta property="og:type" content="article">
<meta property="og:locale" content="en_US">
<meta property="og:site_name" content="">
-<meta property="og:title" content="Cloud Storage">
-<meta property="og:url" content="https://hudi.apache.org/docs/cloud.html";>
+<meta property="og:title" content="COS Filesystem">
+<meta property="og:url" content="https://hudi.apache.org/docs/cos_hoodie.html";>
- <meta property="og:description" content="Talking to Cloud Storage">
+ <meta property="og:description" content="In this page, we explain how to get
your Hudi spark job to store into Tencent Cloud COS.">
- <meta property="article:modified_time" content="2019-06-16T21:59:57-04:00">
+ <meta property="article:modified_time" content="2020-04-21T17:38:24-04:00">
@@ -284,7 +284,7 @@
- <li><a href="/docs/cloud.html" class="active">Cloud
Storage</a></li>
+ <li><a href="/docs/cloud.html" class="">Cloud Storage</a></li>
@@ -353,7 +353,7 @@
<div class="page__inner-wrap">
<header>
- <h1 id="page-title" class="page__title" itemprop="headline">Cloud
Storage
+ <h1 id="page-title" class="page__title" itemprop="headline">COS
Filesystem
</h1>
<!-- Output author details if some exist. -->
@@ -362,29 +362,79 @@
<section class="page__content" itemprop="text">
- <aside class="sidebar__right sticky">
- <nav class="toc">
- <header><h4 class="nav__title"><i class="fas fa-file-alt"></i> IN
THIS PAGE</h4></header>
- <ul class="toc__menu">
- <li><a href="#talking-to-cloud-storage">Talking to Cloud Storage</a></li>
-</ul>
- </nav>
- </aside>
+ <style>
+ .page {
+ padding-right: 0 !important;
+ }
+ </style>
- <h2 id="talking-to-cloud-storage">Talking to Cloud Storage</h2>
+ <p>In this page, we explain how to get your Hudi spark job to store
into Tencent Cloud COS.</p>
+
+<h2 id="tencent-cloud-cos-configs">Tencent Cloud COS configs</h2>
+
+<p>There are two configurations required for Hudi-COS compatibility:</p>
+
+<ul>
+ <li>Adding Tencent Cloud COS Credentials for Hudi</li>
+ <li>Adding required Jars to classpath</li>
+</ul>
-<p>Immaterial of whether RDD/WriteClient APIs or Datasource is used, the
following information helps configure access
-to cloud stores.</p>
+<h3 id="tencent-cloud-cos-credentials">Tencent Cloud COS Credentials</h3>
+
+<p>Add the required configs in your core-site.xml from where Hudi can fetch
them. Replace the <code class="highlighter-rouge">fs.defaultFS</code> with your
COS bucket name, replace <code
class="highlighter-rouge">fs.cosn.userinfo.secretId</code> with your COS secret
Id, replace <code class="highlighter-rouge">fs.cosn.userinfo.secretKey</code>
with your COS key. Hudi should be able to read/write from the bucket.</p>
+
+<div class="language-xml highlighter-rouge"><div class="highlight"><pre
class="highlight"><code> <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.defaultFS<span
class="nt"></name></span>
+ <span class="nt"><value></span>cosn://bucketname<span
class="nt"></value></span>
+ <span class="nt"><description></span>COS bucket name<span
class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.cosn.userinfo.secretId<span
class="nt"></name></span>
+ <span class="nt"><value></span>cos-secretId<span
class="nt"></value></span>
+ <span class="nt"><description></span>Tencent Cloud Secret
Id<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.cosn.userinfo.secretKey<span
class="nt"></name></span>
+ <span class="nt"><value></span>cos-secretkey<span
class="nt"></value></span>
+ <span class="nt"><description></span>Tencent Cloud Secret
Key<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.cosn.bucket.region<span
class="nt"></name></span>
+ <span class="nt"><value></span>ap-region<span
class="nt"></value></span>
+ <span class="nt"><description></span>The region where the bucket
is located.<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span
class="nt"><name></span>fs.cosn.bucket.endpoint_suffix<span
class="nt"></name></span>
+ <span class="nt"><value></span>cos.endpoint.suffix<span
class="nt"></value></span>
+ <span class="nt"><description></span>
+ COS endpoint to connect to.
+ For public cloud users, it is recommended not to set this option,
and only the correct area field is required.
+ <span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span class="nt"><name></span>fs.cosn.impl<span
class="nt"></name></span>
+ <span
class="nt"><value></span>org.apache.hadoop.fs.CosFileSystem<span
class="nt"></value></span>
+ <span class="nt"><description></span>The implementation class of
the CosN Filesystem.<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+ <span class="nt"><property></span>
+ <span
class="nt"><name></span>fs.AbstractFileSystem.cosn.impl<span
class="nt"></name></span>
+ <span class="nt"><value></span>org.apache.hadoop.fs.CosN<span
class="nt"></value></span>
+ <span class="nt"><description></span>The implementation class of
the CosN AbstractFileSystem.<span class="nt"></description></span>
+ <span class="nt"></property></span>
+
+</code></pre></div></div>
+
+<h3 id="tencent-cloud-cos-libs">Tencent Cloud COS Libs</h3>
+<p>COS hadoop libraries to add to our classpath</p>
<ul>
- <li><a href="/docs/s3_hoodie">AWS S3</a> <br />
-Configurations required for S3 and Hudi co-operability.</li>
- <li><a href="/docs/gcs_hoodie">Google Cloud Storage</a> <br />
-Configurations required for GCS and Hudi co-operability.</li>
- <li><a href="/docs/oss_hoodie.html">Alibaba Cloud OSS</a> <br />
-Configurations required for OSS and Hudi co-operability.</li>
- <li><a href="/docs/azure_hoodie.html">Microsoft Azure</a> <br />
-Configurations required for Azure and Hudi co-operability.</li>
+ <li>org.apache.hadoop:hadoop-cos:2.8.5</li>
</ul>
</section>
diff --git a/content/sitemap.xml b/content/sitemap.xml
index 1d0b7e4..c37e854 100644
--- a/content/sitemap.xml
+++ b/content/sitemap.xml
@@ -601,6 +601,14 @@
<lastmod>2020-05-25T19:00:57-04:00</lastmod>
</url>
<url>
+<loc>https://hudi.apache.org/cn/docs/cos_hoodie.html</loc>
+<lastmod>2020-04-21T18:50:50-04:00</lastmod>
+</url>
+<url>
+<loc>https://hudi.apache.org/docs/cos_hoodie.html</loc>
+<lastmod>2020-04-21T17:38:24-04:00</lastmod>
+</url>
+<url>
<loc>https://hudi.apache.org/cn/docs/quick-start-guide.html</loc>
<lastmod>2019-12-30T14:59:57-05:00</lastmod>
</url>
![https://hudi.apache.org/assets/images/500x300.png"},{](https://hudi.apache.org/assets/images/500x300.png"},{){kind=link}