This is an automated email from the ASF dual-hosted git repository. silver pushed a commit to branch services-doc in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 421e4f87f32a22ad2716cf2cf6ce45d16ecde2ce Author: silver-ymz <[email protected]> AuthorDate: Fri Aug 25 18:36:59 2023 +0800 docs: migrate all existed service documents Signed-off-by: silver-ymz <[email protected]> --- core/src/services/fs/docs.md | 4 +- core/src/services/http/backend.rs | 52 +----- core/src/services/http/docs.md | 48 ++++++ core/src/services/memcached/backend.rs | 45 +----- core/src/services/memcached/docs.md | 44 +++++ core/src/services/obs/backend.rs | 58 +------ core/src/services/obs/docs.md | 55 +++++++ core/src/services/supabase/backend.rs | 52 +----- core/src/services/supabase/docs.md | 53 ++++++ core/src/services/wasabi/backend.rs | 223 +------------------------- core/src/services/wasabi/docs.md | 198 +++++++++++++++++++++++ core/src/services/webdav/backend.rs | 53 +----- core/src/services/webdav/docs.md | 51 ++++++ website/docs/services/gcs.mdx | 1 + website/docs/services/http.mdx | 58 +++++++ website/docs/services/memcached.mdx | 58 +++++++ website/docs/services/obs.mdx | 64 ++++++++ website/docs/services/supabase.mdx | 67 ++++++++ website/docs/services/{gcs.mdx => wasabi.mdx} | 37 ++--- website/docs/services/webdav.mdx | 64 ++++++++ 20 files changed, 790 insertions(+), 495 deletions(-) diff --git a/core/src/services/fs/docs.md b/core/src/services/fs/docs.md index 0d5517c63..5547a17af 100644 --- a/core/src/services/fs/docs.md +++ b/core/src/services/fs/docs.md @@ -18,8 +18,8 @@ This service can be used to: ## Configuration - `root`: Set the work dir for backend. - -Refer to public API docs for more information. +- +You can refer to [`FsBuilder`]'s docs for more information ## Example diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs index 25f06f993..76b98fb73 100644 --- a/core/src/services/http/backend.rs +++ b/core/src/services/http/backend.rs @@ -32,56 +32,8 @@ use super::error::parse_error; use crate::raw::*; use crate::*; -/// HTTP Read-only service support like Nginx and Caddy. -/// -/// # Capabilities -/// -/// This service can be used to: -/// -/// - [x] stat -/// - [x] read -/// - [ ] ~~write~~ -/// - [ ] ~~create_dir~~ -/// - [ ] ~~delete~~ -/// - [ ] ~~copy~~ -/// - [ ] ~~rename~~ -/// - [ ] ~~list~~ -/// - [ ] ~~scan~~ -/// - [ ] ~~presign~~ -/// - [ ] blocking -/// -/// # Notes -/// -/// Only `read` ans `stat` are supported. We can use this service to visit any -/// HTTP Server like nginx, caddy. -/// -/// # Configuration -/// -/// - `endpoint`: set the endpoint for http -/// - `root`: Set the work directory for backend -/// -/// You can refer to [`HttpBuilder`]'s docs for more information -/// -/// # Example -/// -/// ## Via Builder -/// -/// ```no_run -/// use anyhow::Result; -/// use opendal::services::Http; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// // create backend builder -/// let mut builder = Http::default(); -/// -/// builder.endpoint("127.0.0.1"); -/// -/// let op: Operator = Operator::new(builder)?.finish(); -/// Ok(()) -/// } -/// ``` +/// HTTP Read-only service support like [Nginx](https://www.nginx.com/) and [Caddy](https://caddyserver.com/). +#[doc = include_str!("docs.md")] #[derive(Default)] pub struct HttpBuilder { endpoint: Option<String>, diff --git a/core/src/services/http/docs.md b/core/src/services/http/docs.md new file mode 100644 index 000000000..8e572214a --- /dev/null +++ b/core/src/services/http/docs.md @@ -0,0 +1,48 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [ ] ~~write~~ +- [ ] ~~create_dir~~ +- [ ] ~~delete~~ +- [ ] ~~copy~~ +- [ ] ~~rename~~ +- [ ] ~~list~~ +- [ ] ~~scan~~ +- [ ] ~~presign~~ +- [ ] blocking + +## Notes + +Only `read` ans `stat` are supported. We can use this service to visit any +HTTP Server like nginx, caddy. + +## Configuration + +- `endpoint`: set the endpoint for http +- `root`: Set the work directory for backend + +You can refer to [`HttpBuilder`]'s docs for more information + +## Example + +### Via Builder + +```rust +use anyhow::Result; +use opendal::services::Http; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + // create http backend builder + let mut builder = Http::default(); + + builder.endpoint("127.0.0.1"); + + let op: Operator = Operator::new(builder)?.finish(); + Ok(()) +} +``` \ No newline at end of file diff --git a/core/src/services/memcached/backend.rs b/core/src/services/memcached/backend.rs index 23fad5b50..2731a70a1 100644 --- a/core/src/services/memcached/backend.rs +++ b/core/src/services/memcached/backend.rs @@ -29,50 +29,7 @@ use crate::raw::*; use crate::*; /// [Memcached](https://memcached.org/) service support. -/// -/// # Capabilities -/// -/// This service can be used to: -/// -/// - [x] stat -/// - [x] read -/// - [x] write -/// - [x] create_dir -/// - [x] delete -/// - [ ] copy -/// - [ ] rename -/// - [ ] ~~list~~ -/// - [ ] scan -/// - [ ] ~~presign~~ -/// - [ ] blocking -/// -/// # Configuration -/// -/// - `root`: Set the working directory of `OpenDAL` -/// - `endpoint`: Set the network address of memcached server -/// - `default_ttl`: Set the ttl for memcached service. -/// -/// You can refer to [`MemcachedBuilder`]'s docs for more information -/// -/// # Example -/// -/// ## Via Builder -/// -/// ```no_run -/// use anyhow::Result; -/// use opendal::services::Memcached; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = Memcached::default(); -/// -/// builder.endpoint("tcp://127.0.0.1:11211"); -/// -/// let op: Operator = Operator::new(builder)?.finish(); -/// Ok(()) -/// } -/// ``` +#[doc = include_str!("docs.md")] #[derive(Clone, Default)] pub struct MemcachedBuilder { /// network address of the memcached service. diff --git a/core/src/services/memcached/docs.md b/core/src/services/memcached/docs.md new file mode 100644 index 000000000..0de179ab9 --- /dev/null +++ b/core/src/services/memcached/docs.md @@ -0,0 +1,44 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [x] write +- [x] create_dir +- [x] delete +- [ ] copy +- [ ] rename +- [ ] ~~list~~ +- [ ] scan +- [ ] ~~presign~~ +- [ ] blocking + +## Configuration + +- `root`: Set the working directory of `OpenDAL` +- `endpoint`: Set the network address of memcached server +- `default_ttl`: Set the ttl for memcached service. + +You can refer to [`MemcachedBuilder`]'s docs for more information + +## Example + +### Via Builder + +```rust +use anyhow::Result; +use opendal::services::Memcached; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + // create memcached backend builder + let mut builder = Memcached::default(); + + builder.endpoint("tcp://127.0.0.1:11211"); + + let op: Operator = Operator::new(builder)?.finish(); + Ok(()) +} +``` \ No newline at end of file diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 78ecf0976..123f5b818 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -41,64 +41,8 @@ use crate::*; /// ref: <https://support.huaweicloud.com/intl/en-us/ugobs-obs/obs_41_0021.html> const MINIMUM_MULTIPART_SIZE: usize = 5 * 1024 * 1024; -/// Huawei Cloud OBS services support. -/// -/// # Capabilities -/// -/// This service can be used to: -/// -/// - [x] stat -/// - [x] read -/// - [x] write -/// - [x] create_dir -/// - [x] delete -/// - [x] copy -/// - [ ] rename -/// - [x] list -/// - [x] scan -/// - [x] presign -/// - [ ] blocking -/// -/// # Configuration -/// -/// - `root`: Set the work directory for backend -/// - `bucket`: Set the container name for backend -/// - `endpoint`: Customizable endpoint setting -/// - `access_key_id`: Set the access_key_id for backend. -/// - `secret_access_key`: Set the secret_access_key for backend. -/// -/// You can refer to [`ObsBuilder`]'s docs for more information -/// -/// # Example -/// -/// ## Via Builder -/// -/// ```no_run -/// use anyhow::Result; -/// use opendal::services::Obs; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// // create backend builder -/// let mut builder = Obs::default(); -/// -/// // set the storage bucket for OpenDAL -/// builder.bucket("test"); -/// // Set the access_key_id and secret_access_key. -/// // -/// // OpenDAL will try load credential from the env. -/// // If credential not set and no valid credential in env, OpenDAL will -/// // send request without signing like anonymous user. -/// builder.access_key_id("access_key_id"); -/// builder.secret_access_key("secret_access_key"); -/// -/// let op: Operator = Operator::new(builder)?.finish(); -/// -/// Ok(()) -/// } -/// ``` /// Huawei-Cloud Object Storage Service (OBS) support +#[doc = include_str!("docs.md")] #[derive(Default, Clone)] pub struct ObsBuilder { root: Option<String>, diff --git a/core/src/services/obs/docs.md b/core/src/services/obs/docs.md new file mode 100644 index 000000000..d080ed585 --- /dev/null +++ b/core/src/services/obs/docs.md @@ -0,0 +1,55 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [x] write +- [x] create_dir +- [x] delete +- [x] copy +- [ ] rename +- [x] list +- [x] scan +- [x] presign +- [ ] blocking + +## Configuration + +- `root`: Set the work directory for backend +- `bucket`: Set the container name for backend +- `endpoint`: Customizable endpoint setting +- `access_key_id`: Set the access_key_id for backend. +- `secret_access_key`: Set the secret_access_key for backend. + +You can refer to [`ObsBuilder`]'s docs for more information + +## Example + +### Via Builder + +```rust +use anyhow::Result; +use opendal::services::Obs; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + // create backend builder + let mut builder = Obs::default(); + + // set the storage bucket for OpenDAL + builder.bucket("test"); + // Set the access_key_id and secret_access_key. + // + // OpenDAL will try load credential from the env. + // If credential not set and no valid credential in env, OpenDAL will + // send request without signing like anonymous user. + builder.access_key_id("access_key_id"); + builder.secret_access_key("secret_access_key"); + + let op: Operator = Operator::new(builder)?.finish(); + + Ok(()) +} +``` \ No newline at end of file diff --git a/core/src/services/supabase/backend.rs b/core/src/services/supabase/backend.rs index 3872623c2..2ace7385c 100644 --- a/core/src/services/supabase/backend.rs +++ b/core/src/services/supabase/backend.rs @@ -28,56 +28,8 @@ use super::writer::*; use crate::raw::*; use crate::*; -/// Supabase service -/// -/// # Capabilities -/// -/// - [x] stat -/// - [x] read -/// - [x] write -/// - [x] create_dir -/// - [x] delete -/// - [ ] copy -/// - [ ] rename -/// - [ ] list -/// - [ ] scan -/// - [ ] presign -/// - [ ] blocking -/// -/// # Configuration -/// -/// - `root`: Set the work dir for backend. -/// - `bucket`: Set the container name for backend. -/// - `endpoint`: Set the endpoint for backend. -/// - `key`: Set the authorization key for the backend, do not set if you want to read public bucket -/// -/// ## Authorization keys -/// -/// There are two types of key in the Supabase, one is anon_key(Client key), another one is -/// service_role_key(Secret key). The former one can only write public resources while the latter one -/// can access all resources. Note that if you want to read public resources, do not set the key. -/// -/// # Example -/// -/// ```no_run -/// use anyhow::Result; -/// use opendal::services::Supabase; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = Supabase::default(); -/// builder.root("/"); -/// builder.bucket("test_bucket"); -/// builder.endpoint("http://127.0.0.1:54321";); -/// // this sets up the anon_key, which means this operator can only write public resource -/// builder.key("some_anon_key"); -/// -/// let op: Operator = Operator::new(builder)?.finish(); -/// -/// Ok(()) -/// } -/// ``` +/// [Supabase](https://supabase.com/) service support +#[doc = include_str!("docs.md")] #[derive(Default)] pub struct SupabaseBuilder { root: Option<String>, diff --git a/core/src/services/supabase/docs.md b/core/src/services/supabase/docs.md new file mode 100644 index 000000000..331ae7fd6 --- /dev/null +++ b/core/src/services/supabase/docs.md @@ -0,0 +1,53 @@ +## Capabilities + +- [x] stat +- [x] read +- [x] write +- [x] create_dir +- [x] delete +- [ ] copy +- [ ] rename +- [ ] list +- [ ] scan +- [ ] presign +- [ ] blocking + +## Configuration + +- `root`: Set the work dir for backend. +- `bucket`: Set the container name for backend. +- `endpoint`: Set the endpoint for backend. +- `key`: Set the authorization key for the backend, do not set if you want to read public bucket + +### Authorization keys + +There are two types of key in the Supabase, one is anon_key(Client key), another one is +service_role_key(Secret key). The former one can only write public resources while the latter one +can access all resources. Note that if you want to read public resources, do not set the key. + +## Example + +### Via Builder + +```rust +use anyhow::Result; +use opendal::services::Supabase; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = Supabase::default(); + + builder.root("/"); + builder.bucket("test_bucket"); + builder.endpoint("http://127.0.0.1:54321";); + // this sets up the anon_key, which means this operator can only write public resource + builder.key("some_anon_key"); + + let op: Operator = Operator::new(builder)?.finish(); + + Ok(()) +} +``` + + \ No newline at end of file diff --git a/core/src/services/wasabi/backend.rs b/core/src/services/wasabi/backend.rs index a8a8da91c..a08381616 100644 --- a/core/src/services/wasabi/backend.rs +++ b/core/src/services/wasabi/backend.rs @@ -55,228 +55,7 @@ static ENDPOINT_TEMPLATES: Lazy<HashMap<&'static str, &'static str>> = Lazy::new }); /// Wasabi (an aws S3 compatible service) support -/// -/// # Capabilities -/// -/// This service can be used to: -/// -/// - [x] stat -/// - [x] read -/// - [x] write -/// - [x] create_dir -/// - [x] delete -/// - [x] copy -/// - [x] rename -/// - [x] list -/// - [x] scan -/// - [x] presign -/// - [ ] blocking -/// -/// # Configuration -/// -/// - `root`: Set the work dir for backend. -/// - `bucket`: Set the container name for backend. -/// - `endpoint`: Set the endpoint for backend. -/// - `region`: Set the region for backend. -/// - `access_key_id`: Set the access_key_id for backend. -/// - `secret_access_key`: Set the secret_access_key for backend. -/// - `security_token`: Set the security_token for backend. -/// - `default_storage_class`: Set the default storage_class for backend. -/// - `server_side_encryption`: Set the server_side_encryption for backend. -/// - `server_side_encryption_aws_kms_key_id`: Set the server_side_encryption_aws_kms_key_id for backend. -/// - `server_side_encryption_customer_algorithm`: Set the server_side_encryption_customer_algorithm for backend. -/// - `server_side_encryption_customer_key`: Set the server_side_encryption_customer_key for backend. -/// - `server_side_encryption_customer_key_md5`: Set the server_side_encryption_customer_key_md5 for backend. -/// - `disable_config_load`: Disable aws config load from env -/// - `enable_virtual_host_style`: Enable virtual host style. -/// -/// Refer to [`WasabiBuilder`]'s public API docs for more information. -/// -/// # Temporary security credentials -/// -/// OpenDAL now provides support for S3 temporary security credentials in IAM. -/// -/// The way to take advantage of this feature is to build your S3 backend with `Builder::security_token`. -/// -/// But OpenDAL will not refresh the temporary security credentials, please keep in mind to refresh those credentials in time. -/// -/// # Server Side Encryption -/// -/// OpenDAL provides full support of S3 Server Side Encryption(SSE) features. -/// -/// The easiest way to configure them is to use helper functions like -/// -/// - SSE-KMS: `server_side_encryption_with_aws_managed_kms_key` -/// - SSE-KMS: `server_side_encryption_with_customer_managed_kms_key` -/// - SSE-S3: `server_side_encryption_with_s3_key` -/// - SSE-C: `server_side_encryption_with_customer_key` -/// -/// If those functions don't fulfill need, low-level options are also provided: -/// -/// - Use service managed kms key -/// - `server_side_encryption="aws:kms"` -/// - Use customer provided kms key -/// - `server_side_encryption="aws:kms"` -/// - `server_side_encryption_aws_kms_key_id="your-kms-key"` -/// - Use S3 managed key -/// - `server_side_encryption="AES256"` -/// - Use customer key -/// - `server_side_encryption_customer_algorithm="AES256"` -/// - `server_side_encryption_customer_key="base64-of-your-aes256-key"` -/// - `server_side_encryption_customer_key_md5="base64-of-your-aes256-key-md5"` -/// -/// After SSE have been configured, all requests send by this backed will attach those headers. -/// -/// Reference: [Protecting data using server-side encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/serv-side-encryption.html) -/// -/// # Example -/// -/// ## Basic Setup -/// -/// ```no_run -/// use std::sync::Arc; -/// -/// use anyhow::Result; -/// use opendal::services::Wasabi; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// // Create s3 backend builder. -/// let mut builder = Wasabi::default(); -/// // Set the root for s3, all operations will happen under this root. -/// // -/// // NOTE: the root must be absolute path. -/// builder.root("/path/to/dir"); -/// // Set the bucket name, this is required. -/// builder.bucket("test"); -/// // Set the endpoint. -/// // -/// // For examples: -/// // - "https://s3.wasabisys.com"; -/// // - "http://127.0.0.1:9000"; -/// // - "https://oss-ap-northeast-1.aliyuncs.com"; -/// // - "https://cos.ap-seoul.myqcloud.com"; -/// // -/// // Default to "https://s3.wasabisys.com"; -/// builder.endpoint("https://s3.wasabisys.com";); -/// // Set the access_key_id and secret_access_key. -/// // -/// // OpenDAL will try load credential from the env. -/// // If credential not set and no valid credential in env, OpenDAL will -/// // send request without signing like anonymous user. -/// builder.access_key_id("access_key_id"); -/// builder.secret_access_key("secret_access_key"); -/// -/// let op: Operator = Operator::new(builder)?.finish(); -/// -/// Ok(()) -/// } -/// ``` -/// -/// ## Wasabi with SSE-C -/// -/// ```no_run -/// use anyhow::Result; -/// use log::info; -/// use opendal::services::Wasabi; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = Wasabi::default(); -/// -/// // Setup builders -/// -/// // Enable SSE-C -/// builder.server_side_encryption_with_customer_key("AES256", "customer_key".as_bytes()); -/// -/// let op = Operator::new(builder)?.finish(); -/// info!("operator: {:?}", op); -/// -/// // Writing your testing code here. -/// -/// Ok(()) -/// } -/// ``` -/// -/// ## Wasabi with SSE-KMS and aws managed kms key -/// -/// ```no_run -/// use anyhow::Result; -/// use log::info; -/// use opendal::services::Wasabi; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = Wasabi::default(); -/// -/// // Setup builders -/// -/// // Enable SSE-KMS with aws managed kms key -/// builder.server_side_encryption_with_aws_managed_kms_key(); -/// -/// let op = Operator::new(builder)?.finish(); -/// info!("operator: {:?}", op); -/// -/// // Writing your testing code here. -/// -/// Ok(()) -/// } -/// ``` -/// -/// ## Wasabi with SSE-KMS and customer managed kms key -/// -/// ```no_run -/// use anyhow::Result; -/// use log::info; -/// use opendal::services::Wasabi; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = Wasabi::default(); -/// -/// // Setup builders -/// -/// // Enable SSE-KMS with customer managed kms key -/// builder.server_side_encryption_with_customer_managed_kms_key("aws_kms_key_id"); -/// -/// let op = Operator::new(builder)?.finish(); -/// info!("operator: {:?}", op); -/// -/// // Writing your testing code here. -/// -/// Ok(()) -/// } -/// ``` -/// -/// ## Wasabi with SSE-S3 -/// -/// ```no_run -/// use anyhow::Result; -/// use log::info; -/// use opendal::services::Wasabi; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = Wasabi::default(); -/// -/// // Setup builders -/// -/// // Enable SSE-S3 -/// builder.server_side_encryption_with_s3_key(); -/// -/// let op = Operator::new(builder)?.finish(); -/// info!("operator: {:?}", op); -/// -/// // Writing your testing code here. -/// -/// Ok(()) -/// } -/// ``` +#[doc = include_str!("docs.md")] #[derive(Default)] pub struct WasabiBuilder { root: Option<String>, diff --git a/core/src/services/wasabi/docs.md b/core/src/services/wasabi/docs.md new file mode 100644 index 000000000..4b2cb2d42 --- /dev/null +++ b/core/src/services/wasabi/docs.md @@ -0,0 +1,198 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [x] write +- [x] create_dir +- [x] delete +- [x] copy +- [x] rename +- [x] list +- [x] scan +- [x] presign +- [ ] blocking + +## Configuration + +- `root`: Set the work dir for backend. +- `bucket`: Set the container name for backend. +- `endpoint`: Set the endpoint for backend. +- `region`: Set the region for backend. +- `access_key_id`: Set the access_key_id for backend. +- `secret_access_key`: Set the secret_access_key for backend. +- `security_token`: Set the security_token for backend. +- `default_storage_class`: Set the default storage_class for backend. +- `server_side_encryption`: Set the server_side_encryption for backend. +- `server_side_encryption_aws_kms_key_id`: Set the server_side_encryption_aws_kms_key_id for backend. +- `server_side_encryption_customer_algorithm`: Set the server_side_encryption_customer_algorithm for kend. +- `server_side_encryption_customer_key`: Set the server_side_encryption_customer_key for backend. +- `server_side_encryption_customer_key_md5`: Set the server_side_encryption_customer_key_md5 for kend. +- `disable_config_load`: Disable aws config load from env +- `enable_virtual_host_style`: Enable virtual host style. + +Refer to [`WasabiBuilder`]'s public API docs for more information. + +### Temporary security credentials + +OpenDAL now provides support for S3 temporary security credentials in IAM. + +The way to take advantage of this feature is to build your S3 backend with `Builder::security_token`. + +But OpenDAL will not refresh the temporary security credentials, please keep in mind to refresh those entials in time. + +### Server Side Encryption + +OpenDAL provides full support of S3 Server Side Encryption(SSE) features. + +The easiest way to configure them is to use helper functions like + +- SSE-KMS: `server_side_encryption_with_aws_managed_kms_key` +- SSE-KMS: `server_side_encryption_with_customer_managed_kms_key` +- SSE-S3: `server_side_encryption_with_s3_key` +- SSE-C: `server_side_encryption_with_customer_key` + +If those functions don't fulfill need, low-level options are also provided: + +- Use service managed kms key + - `server_side_encryption="aws:kms"` +- Use customer provided kms key + - `server_side_encryption="aws:kms"` + - `server_side_encryption_aws_kms_key_id="your-kms-key"` +- Use S3 managed key + - `server_side_encryption="AES256"` +- Use customer key + - `server_side_encryption_customer_algorithm="AES256"` + - `server_side_encryption_customer_key="base64-of-your-aes256-key"` + - `server_side_encryption_customer_key_md5="base64-of-your-aes256-key-md5"` + +After SSE have been configured, all requests send by this backed will attach those headers. + +Reference: [Protecting data using server-side encryption](https://docs.aws.amazon.com/AmazonS3/latest/rguide/serv-side-encryption.html) + +## Example + +### Via Builder + +#### Basic Setup + +```rust +use anyhow::Result; +use opendal::services::Wasabi; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + // Create s3 backend builder. + let mut builder = Wasabi::default(); + // Set the root for s3, all operations will happen under this root. + // + // NOTE: the root must be absolute path. + builder.root("/path/to/dir"); + // Set the bucket name, this is required. + builder.bucket("test"); + // Set the endpoint. + // + // For examples: + // - "https://s3.wasabisys.com"; + // - "http://127.0.0.1:9000"; + // - "https://oss-ap-northeast-1.aliyuncs.com"; + // - "https://cos.ap-seoul.myqcloud.com"; + // + // Default to "https://s3.wasabisys.com"; + builder.endpoint("https://s3.wasabisys.com";); + // Set the access_key_id and secret_access_key. + // + // OpenDAL will try load credential from the env. + // If credential not set and no valid credential in env, OpenDAL will + // send request without signing like anonymous user. + builder.access_key_id("access_key_id"); + builder.secret_access_key("secret_access_key"); + + let op: Operator = Operator::new(builder)?.finish(); + + Ok(()) +} +``` + +#### Wasabi with SSE-C + +```rust +use anyhow::Result; +use opendal::services::Wasabi; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = Wasabi::default(); + + // Enable SSE-C + builder.server_side_encryption_with_customer_key("AES256", "customer_key".as_bytes()); + + let op = Operator::new(builder)?.finish(); + + Ok(()) +} +``` + +#### Wasabi with SSE-KMS and aws managed kms key + +```rust +use anyhow::Result; +use opendal::services::Wasabi; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = Wasabi::default(); + + // Enable SSE-KMS with aws managed kms key + builder.server_side_encryption_with_aws_managed_kms_key(); + + let op = Operator::new(builder)?.finish(); + + Ok(()) +} +``` + +#### Wasabi with SSE-KMS and customer managed kms key + +```rust +use anyhow::Result; +use opendal::services::Wasabi; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = Wasabi::default(); + + // Enable SSE-KMS with customer managed kms key + builder.server_side_encryption_with_customer_managed_kms_key("aws_kms_key_id"); + + let op = Operator::new(builder)?.finish(); + + Ok(()) +} +``` + +#### Wasabi with SSE-S3 + +```rust +use anyhow::Result; +use log::info; +use opendal::services::Wasabi; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = Wasabi::default(); + + // Enable SSE-S3 + builder.server_side_encryption_with_s3_key(); + + let op = Operator::new(builder)?.finish(); + + Ok(()) +} +``` \ No newline at end of file diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index b4fef1db5..bdf8e9ce4 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -38,58 +38,7 @@ use crate::raw::*; use crate::*; /// [WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) backend support. -/// -/// # Capabilities -/// -/// This service can be used to: -/// -/// - [x] stat -/// - [x] read -/// - [x] write -/// - [x] create_dir -/// - [x] delete -/// - [x] copy -/// - [x] rename -/// - [x] list -/// - [ ] ~~scan~~ -/// - [ ] ~~presign~~ -/// - [ ] blocking -/// -/// # Notes -/// -/// Bazel Remote Caching and Ccache HTTP Storage is also part of this service. -/// Users can use `webdav` to connect those services. -/// -/// # Configuration -/// -/// - `endpoint`: set the endpoint for webdav -/// - `root`: Set the work directory for backend -/// -/// You can refer to [`WebdavBuilder`]'s docs for more information -/// -/// # Example -/// -/// ## Via Builder -/// -/// ```no_run -/// use anyhow::Result; -/// use opendal::services::Webdav; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// // create backend builder -/// let mut builder = Webdav::default(); -/// -/// builder -/// .endpoint("127.0.0.1") -/// .username("xxx") -/// .password("xxx"); -/// -/// let op: Operator = Operator::new(builder)?.finish(); -/// Ok(()) -/// } -/// ``` +#[doc = include_str!("docs.md")] #[derive(Default)] pub struct WebdavBuilder { endpoint: Option<String>, diff --git a/core/src/services/webdav/docs.md b/core/src/services/webdav/docs.md new file mode 100644 index 000000000..67a50ca10 --- /dev/null +++ b/core/src/services/webdav/docs.md @@ -0,0 +1,51 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [x] write +- [x] create_dir +- [x] delete +- [x] copy +- [x] rename +- [x] list +- [ ] ~~scan~~ +- [ ] ~~presign~~ +- [ ] blocking + +## Notes + +Bazel Remote Caching and Ccache HTTP Storage is also part of this service. +Users can use `webdav` to connect those services. + +## Configuration + +- `endpoint`: set the endpoint for webdav +- `root`: Set the work directory for backend + +You can refer to [`WebdavBuilder`]'s docs for more information + +## Example + +### Via Builder + +```rust +use anyhow::Result; +use opendal::services::Webdav; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + // create backend builder + let mut builder = Webdav::default(); + + builder.endpoint("127.0.0.1"); + builder.username("xxx") + builder.password("xxx"); + + let op: Operator = Operator::new(builder)?.finish(); + + Ok(()) +} +``` \ No newline at end of file diff --git a/website/docs/services/gcs.mdx b/website/docs/services/gcs.mdx index 68db48d7d..3a84d4bfa 100644 --- a/website/docs/services/gcs.mdx +++ b/website/docs/services/gcs.mdx @@ -30,6 +30,7 @@ async fn main() -> Result<()> { map.insert("credential".to_string(), "authentication token".to_string()); map.insert("predefined_acl".to_string(), "publicRead".to_string()); map.insert("default_storage_class".to_string(), "STANDARD".to_string()); + let op: Operator = Operator::via_map(Scheme::Gcs, map)?; Ok(()) } diff --git a/website/docs/services/http.mdx b/website/docs/services/http.mdx new file mode 100644 index 000000000..c7fd058ce --- /dev/null +++ b/website/docs/services/http.mdx @@ -0,0 +1,58 @@ +--- +title: Http +--- + +HTTP Read-only service support like [Nginx](https://www.nginx.com/) and [Caddy](https://caddyserver.com/). + +import Docs from '../../../core/src/services/http/docs.md' + +<Docs components={props.components} /> + +### Via Config + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +<Tabs> + <TabItem value="rust" label="Rust" default> + +```rust +use anyhow::Result; +use opendal::Operator; +use opendal::Scheme; +use std::collections::HashMap; + +#[tokio::main] +async fn main() -> Result<()> { + let mut map = HashMap::new(); + map.insert("endpoint".to_string(), "127.0.0.1".to_string()); + + let op: Operator = Operator::via_map(Scheme::Http, map)?; + Ok(()) +} +``` + + </TabItem> + <TabItem value="node.js" label="Node.js"> + +```javascript +import { Operator } from "opendal"; +async function main() { + const op = new Operator("http", { + endpoint: "127.0.0.1", + }); +} +``` + + </TabItem> + <TabItem value="python" label="Python"> + +```python +import opendal +op = opendal.Operator("http", + endpoint="127.0.0.1", +) +``` + + </TabItem> +</Tabs> \ No newline at end of file diff --git a/website/docs/services/memcached.mdx b/website/docs/services/memcached.mdx new file mode 100644 index 000000000..8e3a1d98a --- /dev/null +++ b/website/docs/services/memcached.mdx @@ -0,0 +1,58 @@ +--- +title: Memcached +--- + +[Memcached](https://memcached.org/) service support. + +import Docs from '../../../core/src/services/memcached/docs.md' + +<Docs components={props.components} /> + +### Via Config + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +<Tabs> + <TabItem value="rust" label="Rust" default> + +```rust +use anyhow::Result; +use opendal::Operator; +use opendal::Scheme; +use std::collections::HashMap; + +#[tokio::main] +async fn main() -> Result<()> { + let mut map = HashMap::new(); + map.insert("endpoint".to_string(), "tcp://127.0.0.1:11211".to_string()); + + let op: Operator = Operator::via_map(Scheme::Memcached, map)?; + Ok(()) +} +``` + + </TabItem> + <TabItem value="node.js" label="Node.js"> + +```javascript +import { Operator } from "opendal"; +async function main() { + const op = new Operator("memcached", { + endpoint: "tcp://127.0.0.1:11211", + }); +} +``` + + </TabItem> + <TabItem value="python" label="Python"> + +```python +import opendal +op = opendal.Operator("memcached", + endpoint="tcp://127.0.0.1:11211", +) +``` + + </TabItem> +</Tabs> \ No newline at end of file diff --git a/website/docs/services/obs.mdx b/website/docs/services/obs.mdx new file mode 100644 index 000000000..38eb73fac --- /dev/null +++ b/website/docs/services/obs.mdx @@ -0,0 +1,64 @@ +--- +title: Obs +--- + +Huawei-Cloud Object Storage Service (OBS) support + +import Docs from '../../../core/src/services/obs/docs.md' + +<Docs components={props.components} /> + +### Via Config + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +<Tabs> + <TabItem value="rust" label="Rust" default> + +```rust +use anyhow::Result; +use opendal::Operator; +use opendal::Scheme; +use std::collections::HashMap; + +#[tokio::main] +async fn main() -> Result<()> { + let mut map = HashMap::new(); + map.insert("bucket".to_string(), "test".to_string()); + map.insert("access_key_id".to_string(), "access_key_id".to_string()); + map.insert("secret_access_key".to_string(), "secret_access_key".to_string()); + + let op: Operator = Operator::via_map(Scheme::Obs, map)?; + Ok(()) +} +``` + + </TabItem> + <TabItem value="node.js" label="Node.js"> + +```javascript +import { Operator } from "opendal"; +async function main() { + const op = new Operator("obs", { + bucket: "test", + access_key_id: "access_key_id", + secret_access_key: "secret_access_key", + }); +} +``` + + </TabItem> + <TabItem value="python" label="Python"> + +```python +import opendal +op = opendal.Operator("obs", + bucket="test", + access_key_id="access_key_id", + secret_access_key="secret_access_key", +) +``` + + </TabItem> +</Tabs> \ No newline at end of file diff --git a/website/docs/services/supabase.mdx b/website/docs/services/supabase.mdx new file mode 100644 index 000000000..447f74d38 --- /dev/null +++ b/website/docs/services/supabase.mdx @@ -0,0 +1,67 @@ +--- +title: Supabase +--- + +[Supabase](https://supabase.com/) service support. + +import Docs from '../../../core/src/services/supabase/docs.md' + +<Docs components={props.components} /> + +### Via Config + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +<Tabs> + <TabItem value="rust" label="Rust" default> + +```rust +use anyhow::Result; +use opendal::Operator; +use opendal::Scheme; +use std::collections::HashMap; + +#[tokio::main] +async fn main() -> Result<()> { + let mut map = HashMap::new(); + map.insert("root".to_string(), "/".to_string()); + map.insert("bucket".to_string(), "test_bucket".to_string()); + map.insert("endpoint".to_string(), "http://127.0.0.1:54321".to_string()); + map.insert("key".to_string(), "some_anon_key".to_string()); + + let op: Operator = Operator::via_map(Scheme::Supabase, map)?; + Ok(()) +} +``` + + </TabItem> + <TabItem value="node.js" label="Node.js"> + +```javascript +import { Operator } from "opendal"; +async function main() { + const op = new Operator("supabase", { + root: "/", + bucket: "test_bucket", + endpoint: "http://127.0.0.1:54321";, + key: "some_anon_key", + }); +} +``` + + </TabItem> + <TabItem value="python" label="Python"> + +```python +import opendal +op = opendal.Operator("supabase", + root="/", + bucket="test_bucket", + endpoint="http://127.0.0.1:54321";, + key="some_anon_key", +) +``` + + </TabItem> +</Tabs> \ No newline at end of file diff --git a/website/docs/services/gcs.mdx b/website/docs/services/wasabi.mdx similarity index 53% copy from website/docs/services/gcs.mdx copy to website/docs/services/wasabi.mdx index 68db48d7d..aa7ec677b 100644 --- a/website/docs/services/gcs.mdx +++ b/website/docs/services/wasabi.mdx @@ -1,10 +1,10 @@ --- -title: Gcs +title: Wasabi --- -Google Cloud Storage Support +Wasabi (an aws S3 compatible service) support -import Docs from '../../../core/src/services/gcs/docs.md' +import Docs from '../../../core/src/services/wasabi/docs.md' <Docs components={props.components} /> @@ -25,12 +25,13 @@ use std::collections::HashMap; #[tokio::main] async fn main() -> Result<()> { let mut map = HashMap::new(); - map.insert("bucket".to_string(), "test".to_string()); map.insert("root".to_string(), "/path/to/dir".to_string()); - map.insert("credential".to_string(), "authentication token".to_string()); - map.insert("predefined_acl".to_string(), "publicRead".to_string()); - map.insert("default_storage_class".to_string(), "STANDARD".to_string()); - let op: Operator = Operator::via_map(Scheme::Gcs, map)?; + map.insert("bucket".to_string(), "test".to_string()); + map.insert("endpoint".to_string(), "https://s3.wasabisys.com".to_string()); + map.insert("access_key_id".to_string(), "access_key_id".to_string()); + map.insert("secret_access_key".to_string(), "secret_access_key".to_string()); + + let op: Operator = Operator::via_map(Scheme::Wasabi, map)?; Ok(()) } ``` @@ -41,12 +42,12 @@ async fn main() -> Result<()> { ```javascript import { Operator } from "opendal"; async function main() { - const op = new Operator("gcs", { - bucket: "test", + const op = new Operator("wasabi", { root: "/path/to/dir", - credential: "authentication token", - predefined_acl: "publicRead", - default_storage_class: "STANDARD", + bucket: "test", + endpoint: "https://s3.wasabisys.com";, + access_key_id: "access_key_id", + secret_access_key: "secret_access_key", }); } ``` @@ -56,12 +57,12 @@ async function main() { ```python import opendal -op = opendal.Operator("gcs", - bucket="test", +op = opendal.Operator("wasabi", root="/path/to/dir", - credential="authentication token", - predefined_acl="publicRead", - default_storage_class="STANDARD", + bucket="test", + endpoint="https://s3.wasabisys.com";, + access_key_id="access_key_id", + secret_access_key="secret_access_key", ) ``` diff --git a/website/docs/services/webdav.mdx b/website/docs/services/webdav.mdx new file mode 100644 index 000000000..ce0cd2c6f --- /dev/null +++ b/website/docs/services/webdav.mdx @@ -0,0 +1,64 @@ +--- +title: WebDAV +--- + +[WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) backend support. + +import Docs from '../../../core/src/services/webdav/docs.md' + +<Docs components={props.components} /> + +### Via Config + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +<Tabs> + <TabItem value="rust" label="Rust" default> + +```rust +use anyhow::Result; +use opendal::Operator; +use opendal::Scheme; +use std::collections::HashMap; + +#[tokio::main] +async fn main() -> Result<()> { + let mut map = HashMap::new(); + map.insert("endpoint".to_string(), "127.0.0.1".to_string()); + map.insert("username".to_string(), "xxx".to_string()); + map.insert("password".to_string(), "xxx".to_string()); + + let op: Operator = Operator::via_map(Scheme::Webdav, map)?; + Ok(()) +} +``` + + </TabItem> + <TabItem value="node.js" label="Node.js"> + +```javascript +import { Operator } from "opendal"; +async function main() { + const op = new Operator("webdav", { + endpoint: "127.0.0.1", + username: "xxx", + password: "xxx", + }); +} +``` + + </TabItem> + <TabItem value="python" label="Python"> + +```python +import opendal +op = opendal.Operator("webdav", + endpoint="127.0.0.1", + username="xxx", + password="xxx", +) +``` + + </TabItem> +</Tabs> \ No newline at end of file
