This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/asf-site by this push:
new f9d0dc9b9a [DOCS] Update s3_hoodie.md (#6728)
f9d0dc9b9a is described below
commit f9d0dc9b9a56d1084e04ce91cd036a1651b2dad2
Author: Matt Sarrel <[email protected]>
AuthorDate: Fri Sep 23 23:10:42 2022 -0700
[DOCS] Update s3_hoodie.md (#6728)
Co-authored-by: Y Ethan Guo <[email protected]>
---
website/docs/s3_hoodie.md | 39 +++++++++++---------
website/versioned_docs/version-0.12.0/s3_hoodie.md | 41 +++++++++++++---------
2 files changed, 47 insertions(+), 33 deletions(-)
diff --git a/website/docs/s3_hoodie.md b/website/docs/s3_hoodie.md
index a67de25b82..044b201c7c 100644
--- a/website/docs/s3_hoodie.md
+++ b/website/docs/s3_hoodie.md
@@ -15,39 +15,49 @@ There are two configurations required for Hudi-S3
compatibility:
### AWS Credentials
-Simplest way to use Hudi with S3, is to configure your `SparkSession` or
`SparkContext` with S3 credentials. Hudi will automatically pick this up and
talk to S3.
+The simplest way to use Hudi with S3, is to configure your `SparkSession` or
`SparkContext` with S3 credentials. Hudi will automatically pick this up and
talk to S3.
Alternatively, add the required configs in your core-site.xml from where Hudi
can fetch them. Replace the `fs.defaultFS` with your S3 bucket name and Hudi
should be able to read/write from the bucket.
```xml
<property>
- <name>fs.defaultFS</name>
- <value>s3://ysharma</value>
+ <name>fs.defaultFS</name>
+ <value>s3://ysharma</value>
</property>
<property>
- <name>fs.s3.impl</name>
- <value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value>
+ <name>fs.s3.awsAccessKeyId</name>
+ <value>AWS_KEY</value>
</property>
<property>
- <name>fs.s3.awsAccessKeyId</name>
- <value>AWS_KEY</value>
+ <name>fs.s3.awsSecretAccessKey</name>
+ <value>AWS_SECRET</value>
</property>
<property>
- <name>fs.s3.awsSecretAccessKey</name>
- <value>AWS_SECRET</value>
+ <name>fs.s3a.awsAccessKeyId</name>
+ <value>AWS_KEY</value>
</property>
<property>
- <name>fs.s3n.awsAccessKeyId</name>
- <value>AWS_KEY</value>
+ <name>fs.s3a.awsSecretAccessKey</name>
+ <value>AWS_SECRET</value>
</property>
<property>
- <name>fs.s3n.awsSecretAccessKey</name>
- <value>AWS_SECRET</value>
+ <name>fs.s3a.endpoint</name>
+ <value>http://IP-Address:Port</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.path.style.access</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.signing-algorithm</name>
+ <value>S3SignerType</value>
</property>
```
@@ -60,9 +70,6 @@ export HOODIE_ENV_fs_DOT_s3a_DOT_access_DOT_key=$accessKey
export HOODIE_ENV_fs_DOT_s3a_DOT_secret_DOT_key=$secretKey
export HOODIE_ENV_fs_DOT_s3_DOT_awsAccessKeyId=$accessKey
export HOODIE_ENV_fs_DOT_s3_DOT_awsSecretAccessKey=$secretKey
-export HOODIE_ENV_fs_DOT_s3n_DOT_awsAccessKeyId=$accessKey
-export HOODIE_ENV_fs_DOT_s3n_DOT_awsSecretAccessKey=$secretKey
-export HOODIE_ENV_fs_DOT_s3n_DOT_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
```
diff --git a/website/versioned_docs/version-0.12.0/s3_hoodie.md
b/website/versioned_docs/version-0.12.0/s3_hoodie.md
index a67de25b82..876afc0465 100644
--- a/website/versioned_docs/version-0.12.0/s3_hoodie.md
+++ b/website/versioned_docs/version-0.12.0/s3_hoodie.md
@@ -15,39 +15,49 @@ There are two configurations required for Hudi-S3
compatibility:
### AWS Credentials
-Simplest way to use Hudi with S3, is to configure your `SparkSession` or
`SparkContext` with S3 credentials. Hudi will automatically pick this up and
talk to S3.
+The simplest way to use Hudi with S3, is to configure your `SparkSession` or
`SparkContext` with S3 credentials. Hudi will automatically pick this up and
talk to S3.
Alternatively, add the required configs in your core-site.xml from where Hudi
can fetch them. Replace the `fs.defaultFS` with your S3 bucket name and Hudi
should be able to read/write from the bucket.
```xml
<property>
- <name>fs.defaultFS</name>
- <value>s3://ysharma</value>
+ <name>fs.defaultFS</name>
+ <value>s3://ysharma</value>
</property>
<property>
- <name>fs.s3.impl</name>
- <value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value>
+ <name>fs.s3.awsAccessKeyId</name>
+ <value>AWS_KEY</value>
</property>
<property>
- <name>fs.s3.awsAccessKeyId</name>
- <value>AWS_KEY</value>
+ <name>fs.s3.awsSecretAccessKey</name>
+ <value>AWS_SECRET</value>
</property>
<property>
- <name>fs.s3.awsSecretAccessKey</name>
- <value>AWS_SECRET</value>
+ <name>fs.s3a.awsAccessKeyId</name>
+ <value>AWS_KEY</value>
</property>
<property>
- <name>fs.s3n.awsAccessKeyId</name>
- <value>AWS_KEY</value>
+ <name>fs.s3a.awsSecretAccessKey</name>
+ <value>AWS_SECRET</value>
</property>
<property>
- <name>fs.s3n.awsSecretAccessKey</name>
- <value>AWS_SECRET</value>
+ <name>fs.s3a.endpoint</name>
+ <value>http://IP-Address:Port</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.path.style.access</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.signing-algorithm</name>
+ <value>S3SignerType</value>
</property>
```
@@ -60,9 +70,6 @@ export HOODIE_ENV_fs_DOT_s3a_DOT_access_DOT_key=$accessKey
export HOODIE_ENV_fs_DOT_s3a_DOT_secret_DOT_key=$secretKey
export HOODIE_ENV_fs_DOT_s3_DOT_awsAccessKeyId=$accessKey
export HOODIE_ENV_fs_DOT_s3_DOT_awsSecretAccessKey=$secretKey
-export HOODIE_ENV_fs_DOT_s3n_DOT_awsAccessKeyId=$accessKey
-export HOODIE_ENV_fs_DOT_s3n_DOT_awsSecretAccessKey=$secretKey
-export HOODIE_ENV_fs_DOT_s3n_DOT_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
```
@@ -84,4 +91,4 @@ AWS glue data libraries are needed if AWS glue data is used
With versioned buckets any object deleted creates a [Delete
Marker](https://docs.aws.amazon.com/AmazonS3/latest/userguide/DeleteMarker.html),
as Hudi cleans up files using [Cleaner
utility](https://hudi.apache.org/docs/hoodie_cleaner) the number of Delete
Markers increases over time.
It is important to configure the [Lifecycle
Rule](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lifecycle-mgmt.html)
correctly
to clean up these delete markers as the List operation can choke if the number
of delete markers reaches 1000.
-We recommend cleaning up Delete Markers after 1 day in Lifecycle Rule.
\ No newline at end of file
+We recommend cleaning up Delete Markers after 1 day in Lifecycle Rule.
