This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 958f5453 test(core): test for append (#2286)
958f5453 is described below
commit 958f5453637b46f587675696c12848f142c55ac4
Author: Suyan <[email protected]>
AuthorDate: Mon May 22 23:05:45 2023 +0800
test(core): test for append (#2286)
* test(core): test for append
Signed-off-by: suyanhanx <[email protected]>
* merge append_to_non_exsiting_target to append
Signed-off-by: suyanhanx <[email protected]>
---------
Signed-off-by: suyanhanx <[email protected]>
---
core/src/types/capability.rs | 6 +
core/src/types/operator/metadata.rs | 5 +
core/tests/behavior/append.rs | 223 ++++++++++++++++++++++++++++++++++++
core/tests/behavior/main.rs | 4 +
core/tests/behavior/write.rs | 30 +++++
5 files changed, 268 insertions(+)
diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs
index 14685c30..c6b8e981 100644
--- a/core/src/types/capability.rs
+++ b/core/src/types/capability.rs
@@ -88,6 +88,12 @@ pub struct Capability {
/// If operator supports append natively, it will be true.
pub append: bool,
+ /// If operator supports append with content type natively, it will be
true.
+ pub append_with_content_type: bool,
+ /// If operator supports append with content disposition natively, it will
be true.
+ pub append_with_content_disposition: bool,
+ /// If operator supports append with cache control natively, it will be
true.
+ pub append_with_cache_control: bool,
/// If operator supports create dir natively, it will be true.
pub create_dir: bool,
diff --git a/core/src/types/operator/metadata.rs
b/core/src/types/operator/metadata.rs
index d4d850f7..3d219dfc 100644
--- a/core/src/types/operator/metadata.rs
+++ b/core/src/types/operator/metadata.rs
@@ -62,6 +62,11 @@ impl OperatorInfo {
self.0.capability().write
}
+ /// Check if current backend supports [`Accessor::append`] or not.
+ pub fn can_append(&self) -> bool {
+ self.0.capability().append
+ }
+
/// Check if current backend supports [`Accessor::copy`] or not.
pub fn can_copy(&self) -> bool {
self.0.capability().copy
diff --git a/core/tests/behavior/append.rs b/core/tests/behavior/append.rs
new file mode 100644
index 00000000..1c03a3e6
--- /dev/null
+++ b/core/tests/behavior/append.rs
@@ -0,0 +1,223 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use anyhow::Result;
+use opendal::ops::OpAppend;
+use opendal::EntryMode;
+use opendal::ErrorKind;
+use opendal::Operator;
+
+use super::utils::*;
+
+/// Test services that meet the following capability:
+///
+/// - can_read
+/// - can_write
+/// - can_append
+macro_rules! behavior_append_test {
+ ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
+ paste::item! {
+ $(
+ #[test]
+ $(
+ #[$meta]
+ )*
+ fn [<append_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read()
+ && op.info().can_write()
+ && op.info().can_append() =>
RUNTIME.block_on($crate::append::$test(op.clone())),
+ Some(_) => {
+ log::warn!("service {} doesn't support append,
ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ Ok(())
+ }
+ }
+ }
+ )*
+ }
+ };
+}
+
+#[macro_export]
+macro_rules! behavior_append_tests {
+ ($($service:ident),*) => {
+ $(
+ behavior_append_test!(
+ $service,
+
+ test_append,
+ test_append_with_dir_path,
+ test_append_with_cache_control,
+ test_append_with_content_type,
+ test_append_with_content_disposition,
+
+ test_fuzz_appender,
+ );
+ )*
+ };
+}
+
+/// Test append to a file must success.
+pub async fn test_append(op: Operator) -> Result<()> {
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content_one, size_one) = gen_bytes();
+ let (content_two, size_two) = gen_bytes();
+
+ op.append(&path, content_one.clone())
+ .await
+ .expect("append file first time must success");
+
+ op.append(&path, content_two.clone())
+ .await
+ .expect("append to an existing file must success");
+
+ let bs = op.read(&path).await.expect("read file must success");
+
+ assert_eq!(bs.len(), size_one + size_two);
+ assert_eq!(bs[..size_one], content_one);
+ assert_eq!(bs[size_one..], content_two);
+
+ op.delete(&path).await.expect("delete file must success");
+
+ Ok(())
+}
+
+/// Test append to a directory path must fail.
+pub async fn test_append_with_dir_path(op: Operator) -> Result<()> {
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content, _) = gen_bytes();
+
+ let res = op.append(&path, content).await;
+ assert!(res.is_err());
+ assert_eq!(res.unwrap_err().kind(), ErrorKind::IsADirectory);
+
+ Ok(())
+}
+
+/// Test append with cache control must success.
+pub async fn test_append_with_cache_control(op: Operator) -> Result<()> {
+ if !op.info().capability().write_with_cache_control {
+ return Ok(());
+ }
+
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content, _) = gen_bytes();
+
+ let target_cache_control = "no-cache, no-store, max-age=300";
+
+ let mut op_append = OpAppend::default();
+ op_append = op_append.with_cache_control(target_cache_control);
+
+ op.append_with(&path, op_append, content).await?;
+
+ let meta = op.stat(&path).await.expect("stat must succeed");
+ assert_eq!(meta.mode(), EntryMode::FILE);
+ assert_eq!(
+ meta.cache_control().expect("cache control must exist"),
+ target_cache_control
+ );
+
+ op.delete(&path).await.expect("delete must succeed");
+
+ Ok(())
+}
+
+/// Test append with content type must success.
+pub async fn test_append_with_content_type(op: Operator) -> Result<()> {
+ if !op.info().capability().append_with_content_type {
+ return Ok(());
+ }
+
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content, size) = gen_bytes();
+
+ let target_content_type = "application/json";
+
+ let mut op_append = OpAppend::default();
+ op_append = op_append.with_content_type(target_content_type);
+
+ op.append_with(&path, op_append, content).await?;
+
+ let meta = op.stat(&path).await.expect("stat must succeed");
+ assert_eq!(meta.mode(), EntryMode::FILE);
+ assert_eq!(
+ meta.content_type().expect("content type must exist"),
+ target_content_type
+ );
+ assert_eq!(meta.content_length(), size as u64);
+
+ op.delete(&path).await.expect("delete must succeed");
+
+ Ok(())
+}
+
+/// Write a single file with content disposition should succeed.
+pub async fn test_append_with_content_disposition(op: Operator) -> Result<()> {
+ if !op.info().capability().write_with_content_disposition {
+ return Ok(());
+ }
+
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content, size) = gen_bytes();
+
+ let target_content_disposition = "attachment; filename=\"filename.jpg\"";
+
+ let mut op_append = OpAppend::default();
+ op_append = op_append.with_content_disposition(target_content_disposition);
+
+ op.append_with(&path, op_append, content).await?;
+
+ let meta = op.stat(&path).await.expect("stat must succeed");
+ assert_eq!(meta.mode(), EntryMode::FILE);
+ assert_eq!(
+ meta.content_disposition().expect("content type must exist"),
+ target_content_disposition
+ );
+ assert_eq!(meta.content_length(), size as u64);
+
+ op.delete(&path).await.expect("delete must succeed");
+
+ Ok(())
+}
+
+/// Test for fuzzing appender.
+pub async fn test_fuzz_appender(op: Operator) -> Result<()> {
+ let path = uuid::Uuid::new_v4().to_string();
+
+ let mut fuzzer = ObjectWriterFuzzer::new(&path, None);
+
+ let mut a = op.appender(&path).await?;
+
+ for _ in 0..100 {
+ match fuzzer.fuzz() {
+ ObjectWriterAction::Write(bs) => {
+ a.append(bs).await?;
+ }
+ }
+ }
+ a.close().await?;
+
+ let content = op.read(&path).await?;
+ fuzzer.check(&content);
+
+ op.delete(&path).await.expect("delete file must success");
+
+ Ok(())
+}
diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs
index 3a80f0cd..54fae9e4 100644
--- a/core/tests/behavior/main.rs
+++ b/core/tests/behavior/main.rs
@@ -26,6 +26,8 @@ mod blocking_read;
#[macro_use]
mod blocking_write;
#[macro_use]
+mod append;
+#[macro_use]
mod copy;
#[macro_use]
mod list;
@@ -69,6 +71,8 @@ macro_rules! behavior_tests {
behavior_write_tests!($service);
// can_read && can_write && can_blocking
behavior_blocking_write_tests!($service);
+ // can_read && can_write && can_append
+ behavior_append_tests!($service);
// can_read && can_write && can_copy
behavior_copy_tests!($service);
// can read && can_write && can_blocking && can_copy
diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs
index fa5b47b2..0da8a1d2 100644
--- a/core/tests/behavior/write.rs
+++ b/core/tests/behavior/write.rs
@@ -80,6 +80,7 @@ macro_rules! behavior_write_tests {
test_write_with_special_chars,
test_write_with_cache_control,
test_write_with_content_type,
+ test_write_with_content_disposition,
test_stat,
test_stat_dir,
test_stat_with_special_chars,
@@ -249,6 +250,35 @@ pub async fn test_write_with_content_type(op: Operator) ->
Result<()> {
Ok(())
}
+/// Write a single file with content disposition should succeed.
+pub async fn test_write_with_content_disposition(op: Operator) -> Result<()> {
+ if !op.info().capability().write_with_content_disposition {
+ return Ok(());
+ }
+
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content, size) = gen_bytes();
+
+ let target_content_disposition = "attachment; filename=\"filename.jpg\"";
+
+ let mut op_write = OpWrite::default();
+ op_write = op_write.with_content_disposition(target_content_disposition);
+
+ op.write_with(&path, op_write, content).await?;
+
+ let meta = op.stat(&path).await.expect("stat must succeed");
+ assert_eq!(meta.mode(), EntryMode::FILE);
+ assert_eq!(
+ meta.content_disposition().expect("content type must exist"),
+ target_content_disposition
+ );
+ assert_eq!(meta.content_length(), size as u64);
+
+ op.delete(&path).await.expect("delete must succeed");
+
+ Ok(())
+}
+
/// Stat existing file should return metadata
pub async fn test_stat(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
