This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new ca53000e refactor: Allow reusing the same operator to speed up tests
(#2068)
ca53000e is described below
commit ca53000efe8ac93d00e156548dc697e92649b5cb
Author: Xuanwo <[email protected]>
AuthorDate: Sun Apr 23 11:01:24 2023 +0800
refactor: Allow reusing the same operator to speed up tests (#2068)
* refactor: Allow reusing the same operator to speed up tests
Signed-off-by: Xuanwo <[email protected]>
* Fix scan root
Signed-off-by: Xuanwo <[email protected]>
* Fix debug for cap
Signed-off-by: Xuanwo <[email protected]>
* avoid conflict
Signed-off-by: Xuanwo <[email protected]>
* Fix fs
Signed-off-by: Xuanwo <[email protected]>
* Allow retry content incomplete
Signed-off-by: Xuanwo <[email protected]>
* Allow retry for copy and rename
Signed-off-by: Xuanwo <[email protected]>
* FIx list tests
Signed-off-by: Xuanwo <[email protected]>
* Fix ensure root
Signed-off-by: Xuanwo <[email protected]>
* Allow retry for redis error
Signed-off-by: Xuanwo <[email protected]>
* Refactor random root
Signed-off-by: Xuanwo <[email protected]>
* Use better cap debug format
Signed-off-by: Xuanwo <[email protected]>
* Better!
Signed-off-by: Xuanwo <[email protected]>
* Fix webdav
Signed-off-by: Xuanwo <[email protected]>
* ftp should ensure parent dir exists
Signed-off-by: Xuanwo <[email protected]>
* Refactor tests
Signed-off-by: Xuanwo <[email protected]>
* format code
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
.github/workflows/service_test_http.yml | 2 +
.github/workflows/service_test_ipfs.yml | 2 +
core/src/layers/retry.rs | 28 ++++++++++++
core/src/raw/http_util/body.rs | 6 ++-
core/src/services/ftp/backend.rs | 46 +++++++++++++------
core/src/services/redis/backend.rs | 4 +-
core/src/services/webdav/backend.rs | 4 ++
core/src/services/webdav/pager.rs | 11 +++--
core/src/types/capability.rs | 49 +++++++++++++++++++-
core/tests/behavior/blocking_copy.rs | 41 ++++++++---------
core/tests/behavior/blocking_list.rs | 60 +++++++++++++-----------
core/tests/behavior/blocking_read.rs | 39 ++++++++--------
core/tests/behavior/blocking_rename.rs | 41 ++++++++---------
core/tests/behavior/blocking_write.rs | 39 ++++++++--------
core/tests/behavior/copy.rs | 39 ++++++++--------
core/tests/behavior/list.rs | 81 ++++++++++++++++++---------------
core/tests/behavior/list_only.rs | 35 +++++++-------
core/tests/behavior/main.rs | 68 ++++++++++++++++-----------
core/tests/behavior/presign.rs | 35 +++++++-------
core/tests/behavior/read_only.rs | 35 +++++++-------
core/tests/behavior/rename.rs | 39 ++++++++--------
core/tests/behavior/utils.rs | 6 ++-
core/tests/behavior/write.rs | 35 +++++++-------
23 files changed, 425 insertions(+), 320 deletions(-)
diff --git a/.github/workflows/service_test_http.yml
b/.github/workflows/service_test_http.yml
index 7e3a1ff6..ce32448a 100644
--- a/.github/workflows/service_test_http.yml
+++ b/.github/workflows/service_test_http.yml
@@ -66,6 +66,7 @@ jobs:
RUST_LOG: debug
OPENDAL_HTTP_TEST: on
OPENDAL_HTTP_ENDPOINT: http://127.0.0.1:8080
+ OPENDAL_DISABLE_RANDOM_ROOT: true
caddy:
runs-on: ${{ matrix.os }}
@@ -102,3 +103,4 @@ jobs:
RUST_LOG: debug
OPENDAL_HTTP_TEST: on
OPENDAL_HTTP_ENDPOINT: http://127.0.0.1:8080
+ OPENDAL_DISABLE_RANDOM_ROOT: true
diff --git a/.github/workflows/service_test_ipfs.yml
b/.github/workflows/service_test_ipfs.yml
index ad0edc53..b7138586 100644
--- a/.github/workflows/service_test_ipfs.yml
+++ b/.github/workflows/service_test_ipfs.yml
@@ -63,6 +63,8 @@ jobs:
OPENDAL_IPFS_TEST: on
OPENDAL_IPFS_ROOT:
/ipfs/QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ/
OPENDAL_IPFS_ENDPOINT: "http://127.0.0.1:8080";
+ OPENDAL_DISABLE_RANDOM_ROOT: true
+
# # ipfs.io can't pass our test by now, we should address them in the future.
# ipfs-io:
# runs-on: ubuntu-latest
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index 2193202a..341fd75e 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -247,6 +247,34 @@ impl<A: Accessor> LayeredAccessor for RetryAccessor<A> {
.await
}
+ async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy>
{
+ { || self.inner.copy(from, to, args.clone()) }
+ .retry(&self.builder)
+ .when(|e| e.is_temporary())
+ .notify(|err, dur| {
+ warn!(
+ target: "opendal::service",
+ "operation={} -> retry after {}s: error={:?}",
+ Operation::Copy, dur.as_secs_f64(), err)
+ })
+ .map(|v| v.map_err(|e| e.set_persistent()))
+ .await
+ }
+
+ async fn rename(&self, from: &str, to: &str, args: OpRename) ->
Result<RpRename> {
+ { || self.inner.rename(from, to, args.clone()) }
+ .retry(&self.builder)
+ .when(|e| e.is_temporary())
+ .notify(|err, dur| {
+ warn!(
+ target: "opendal::service",
+ "operation={} -> retry after {}s: error={:?}",
+ Operation::Rename, dur.as_secs_f64(), err)
+ })
+ .map(|v| v.map_err(|e| e.set_persistent()))
+ .await
+ }
+
async fn list(&self, path: &str, args: OpList) -> Result<(RpList,
Self::Pager)> {
{ || self.inner.list(path, args.clone()) }
.retry(&self.builder)
diff --git a/core/src/raw/http_util/body.rs b/core/src/raw/http_util/body.rs
index 4d8eec3f..28905707 100644
--- a/core/src/raw/http_util/body.rs
+++ b/core/src/raw/http_util/body.rs
@@ -146,11 +146,13 @@ impl IncomingAsyncBody {
Ordering::Less => Err(Error::new(
ErrorKind::ContentIncomplete,
&format!("reader got too less data, expect: {expect}, actual:
{actual}"),
- )),
+ )
+ .set_temporary()),
Ordering::Greater => Err(Error::new(
ErrorKind::ContentTruncated,
&format!("reader got too much data, expect: {expect}, actual:
{actual}"),
- )),
+ )
+ .set_temporary()),
}
}
}
diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs
index 0b112c05..32338980 100644
--- a/core/src/services/ftp/backend.rs
+++ b/core/src/services/ftp/backend.rs
@@ -335,22 +335,16 @@ impl Accessor for FtpBackend {
for path in paths {
curr_path.push_str(path);
- // try to create directory
- if curr_path.ends_with('/') {
- match ftp_stream.mkdir(&curr_path).await {
- // Do nothing if status is FileUnavailable or OK(()) is
return.
- Err(FtpError::UnexpectedResponse(Response {
- status: Status::FileUnavailable,
- ..
- }))
- | Ok(()) => (),
- Err(e) => {
- return Err(e.into());
- }
+ match ftp_stream.mkdir(&curr_path).await {
+ // Do nothing if status is FileUnavailable or OK(()) is return.
+ Err(FtpError::UnexpectedResponse(Response {
+ status: Status::FileUnavailable,
+ ..
+ }))
+ | Ok(()) => (),
+ Err(e) => {
+ return Err(e.into());
}
- } else {
- // else, create file
- ftp_stream.put_file(&curr_path, &mut "".as_bytes()).await?;
}
}
@@ -398,6 +392,28 @@ impl Accessor for FtpBackend {
));
}
+ // Ensure the parent dir exists.
+ let parent = get_parent(path);
+ let paths: Vec<&str> = parent.split('/').collect();
+
+ // TODO: we can optimize this by checking dir existence first.
+ let mut ftp_stream = self.ftp_connect(Operation::Write).await?;
+ let mut curr_path = String::new();
+ for path in paths {
+ curr_path.push_str(path);
+ match ftp_stream.mkdir(&curr_path).await {
+ // Do nothing if status is FileUnavailable or OK(()) is return.
+ Err(FtpError::UnexpectedResponse(Response {
+ status: Status::FileUnavailable,
+ ..
+ }))
+ | Ok(()) => (),
+ Err(e) => {
+ return Err(e.into());
+ }
+ }
+ }
+
Ok((
RpWrite::new(),
FtpWriter::new(self.clone(), path.to_string()),
diff --git a/core/src/services/redis/backend.rs
b/core/src/services/redis/backend.rs
index 7d8ac059..3e484af0 100644
--- a/core/src/services/redis/backend.rs
+++ b/core/src/services/redis/backend.rs
@@ -356,6 +356,8 @@ impl kv::Adapter for Adapter {
impl From<RedisError> for Error {
fn from(e: RedisError) -> Self {
- Error::new(ErrorKind::Unexpected, e.category()).set_source(e)
+ Error::new(ErrorKind::Unexpected, e.category())
+ .set_source(e)
+ .set_temporary()
}
}
diff --git a/core/src/services/webdav/backend.rs
b/core/src/services/webdav/backend.rs
index bc1558e0..4a073d6e 100644
--- a/core/src/services/webdav/backend.rs
+++ b/core/src/services/webdav/backend.rs
@@ -648,6 +648,10 @@ impl WebdavBackend {
}
async fn ensure_parent_path(&self, path: &str) -> Result<()> {
+ if path == "/" {
+ return Ok(());
+ }
+
// create dir recursively, split path by `/` and create each dir
except the last one
let abs_path = build_abs_path(&self.root, path);
let abs_path = abs_path.as_str();
diff --git a/core/src/services/webdav/pager.rs
b/core/src/services/webdav/pager.rs
index 32cb5e0f..3cd8ebb8 100644
--- a/core/src/services/webdav/pager.rs
+++ b/core/src/services/webdav/pager.rs
@@ -54,12 +54,13 @@ impl oio::Page for WebdavPager {
.into_iter()
.filter_map(|de| {
let path = de.href;
- let normalized_path = if self.root != path {
- build_rel_path(&self.root, &path)
- } else {
- path
- };
+ // Ignore the root path itself.
+ if self.root == path {
+ return None;
+ }
+
+ let normalized_path = build_rel_path(&self.root, &path);
if normalized_path == self.path {
// WebDav server may return the current path as an entry.
return None;
diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs
index eb6c756b..bb7a872d 100644
--- a/core/src/types/capability.rs
+++ b/core/src/types/capability.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+use std::fmt::Debug;
+
/// Capability is used to describe what operations are supported
/// by current Operator.
///
@@ -42,7 +44,7 @@
/// - Operation with variants should be named like `read_can_seek`.
/// - Operation with arguments should be named like `read_with_range`.
/// - Operation with limtations should be named like `batch_max_operations`.
-#[derive(Copy, Clone, Debug, Default)]
+#[derive(Copy, Clone, Default)]
pub struct Capability {
/// If operator supports stat natively, it will be true.
pub stat: bool,
@@ -127,3 +129,48 @@ pub struct Capability {
/// If operator supports blocking natively, it will be true.
pub blocking: bool,
}
+
+impl Debug for Capability {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let mut s = vec![];
+
+ if self.read {
+ s.push("Read");
+ }
+ if self.stat {
+ s.push("Stat");
+ }
+ if self.write {
+ s.push("Write");
+ }
+ if self.create_dir {
+ s.push("CreateDir");
+ }
+ if self.delete {
+ s.push("Delete");
+ }
+ if self.list {
+ s.push("List");
+ }
+ if self.scan {
+ s.push("Scan");
+ }
+ if self.copy {
+ s.push("Copy");
+ }
+ if self.rename {
+ s.push("Rename");
+ }
+ if self.presign {
+ s.push("Presign");
+ }
+ if self.batch {
+ s.push("Batch");
+ }
+ if self.blocking {
+ s.push("Blocking");
+ }
+
+ write!(f, "{{ {} }}", s.join(" | "))
+ }
+}
diff --git a/core/tests/behavior/blocking_copy.rs
b/core/tests/behavior/blocking_copy.rs
index 8cf724c4..346b8b45 100644
--- a/core/tests/behavior/blocking_copy.rs
+++ b/core/tests/behavior/blocking_copy.rs
@@ -30,31 +30,28 @@ use super::utils::*;
macro_rules! behavior_blocking_copy_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _blocking_copy>] {
+ $(
+ #[test]
$(
- #[test]
- $(
- #[$meta]
- )*
- fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read()
- && op.info().can_write()
- && op.info().can_copy()
- && op.info().can_blocking() =>
$crate::blocking_copy::$test(op.blocking()),
- Some(_) => {
- log::warn!("service {} doesn't support
blocking_copy, ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<blocking_copy_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read()
+ && op.info().can_write()
+ && op.info().can_copy()
+ && op.info().can_blocking() =>
$crate::blocking_copy::$test(op.blocking()),
+ Some(_) => {
+ log::warn!("service {} doesn't support
blocking_copy, ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
diff --git a/core/tests/behavior/blocking_list.rs
b/core/tests/behavior/blocking_list.rs
index c84084d3..20f333e5 100644
--- a/core/tests/behavior/blocking_list.rs
+++ b/core/tests/behavior/blocking_list.rs
@@ -34,30 +34,27 @@ use super::utils::*;
macro_rules! behavior_blocking_list_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _blocking_list>] {
+ $(
+ #[test]
$(
- #[test]
- $(
- #[$meta]
- )*
- fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read()
- && op.info().can_write()
- && op.info().can_blocking() &&
(op.info().can_list()||op.info().can_scan()) =>
$crate::blocking_list::$test(op.blocking()),
- Some(_) => {
- log::warn!("service {} doesn't support
blocking_list, ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<blocking_list_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read()
+ && op.info().can_write()
+ && op.info().can_blocking() &&
(op.info().can_list()||op.info().can_scan()) =>
$crate::blocking_list::$test(op.blocking()),
+ Some(_) => {
+ log::warn!("service {} doesn't support
blocking_list, ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
@@ -79,13 +76,14 @@ macro_rules! behavior_blocking_list_tests {
/// List dir should return newly created file.
pub fn test_list_dir(op: BlockingOperator) -> Result<()> {
- let path = uuid::Uuid::new_v4().to_string();
+ let parent = uuid::Uuid::new_v4().to_string();
+ let path = format!("{parent}/{}", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
let (content, size) = gen_bytes();
op.write(&path, content).expect("write must succeed");
- let obs = op.list("/")?;
+ let obs = op.list(&format!("{parent}/"))?;
let mut found = false;
for de in obs {
let de = de?;
@@ -122,22 +120,30 @@ pub fn test_list_non_exist_dir(op: BlockingOperator) ->
Result<()> {
// Walk top down should output as expected
pub fn test_scan(op: BlockingOperator) -> Result<()> {
+ let parent = uuid::Uuid::new_v4().to_string();
+
let expected = vec![
"x/", "x/y", "x/x/", "x/x/y", "x/x/x/", "x/x/x/y", "x/x/x/x/",
];
for path in expected.iter() {
if path.ends_with('/') {
- op.create_dir(path)?;
+ op.create_dir(&format!("{parent}/{path}"))?;
} else {
- op.write(path, "test_scan")?;
+ op.write(&format!("{parent}/{path}"), "test_scan")?;
}
}
- let w = op.scan("x/")?;
+ let w = op.scan(&format!("{parent}/x/"))?;
let actual = w
.collect::<Vec<_>>()
.into_iter()
- .map(|v| v.unwrap().path().to_string())
+ .map(|v| {
+ v.unwrap()
+ .path()
+ .strip_prefix(&format!("{parent}/"))
+ .unwrap()
+ .to_string()
+ })
.collect::<HashSet<_>>();
debug!("walk top down: {:?}", actual);
diff --git a/core/tests/behavior/blocking_read.rs
b/core/tests/behavior/blocking_read.rs
index 2f4e77a7..2033398c 100644
--- a/core/tests/behavior/blocking_read.rs
+++ b/core/tests/behavior/blocking_read.rs
@@ -30,30 +30,27 @@ use sha2::Sha256;
macro_rules! behavior_blocking_read_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _blocking_read>] {
+ $(
+ #[test]
$(
- #[test]
- $(
- #[$meta]
- )*
- fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read()
- && !op.info().can_write()
- && op.info().can_blocking() =>
$crate::blocking_read::$test(op.blocking()),
- Some(_) => {
- log::warn!("service {} doesn't support
blocking_read, ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<blocking_read_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read()
+ && !op.info().can_write()
+ && op.info().can_blocking() =>
$crate::blocking_read::$test(op.blocking()),
+ Some(_) => {
+ log::warn!("service {} doesn't support
blocking_read, ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
diff --git a/core/tests/behavior/blocking_rename.rs
b/core/tests/behavior/blocking_rename.rs
index 7222b230..3973a954 100644
--- a/core/tests/behavior/blocking_rename.rs
+++ b/core/tests/behavior/blocking_rename.rs
@@ -30,31 +30,28 @@ use super::utils::*;
macro_rules! behavior_blocking_rename_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _blocking_rename>] {
+ $(
+ #[test]
$(
- #[test]
- $(
- #[$meta]
- )*
- fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read()
- && op.info().can_write()
- && op.info().can_rename()
- && op.info().can_blocking() =>
$crate::blocking_rename::$test(op.blocking()),
- Some(_) => {
- log::warn!("service {} doesn't support
blocking_rename, ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<blocking_rename_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read()
+ && op.info().can_write()
+ && op.info().can_rename()
+ && op.info().can_blocking() =>
$crate::blocking_rename::$test(op.blocking()),
+ Some(_) => {
+ log::warn!("service {} doesn't support
blocking_rename, ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
diff --git a/core/tests/behavior/blocking_write.rs
b/core/tests/behavior/blocking_write.rs
index a444b1ca..d1bd392b 100644
--- a/core/tests/behavior/blocking_write.rs
+++ b/core/tests/behavior/blocking_write.rs
@@ -36,30 +36,27 @@ use super::utils::*;
macro_rules! behavior_blocking_write_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _blocking_write>] {
+ $(
+ #[test]
$(
- #[test]
- $(
- #[$meta]
- )*
- fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read()
- && op.info().can_write()
- && op.info().can_blocking() =>
$crate::blocking_write::$test(op.blocking()),
- Some(_) => {
- log::warn!("service {} doesn't support
blocking_write, ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<blocking_write_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read()
+ && op.info().can_write()
+ && op.info().can_blocking() =>
$crate::blocking_write::$test(op.blocking()),
+ Some(_) => {
+ log::warn!("service {} doesn't support
blocking_write, ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
diff --git a/core/tests/behavior/copy.rs b/core/tests/behavior/copy.rs
index e09f6b8d..dfe0a7cc 100644
--- a/core/tests/behavior/copy.rs
+++ b/core/tests/behavior/copy.rs
@@ -29,30 +29,27 @@ use super::utils::*;
macro_rules! behavior_copy_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _copy>] {
+ $(
+ #[test]
$(
- #[tokio::test]
- $(
- #[$meta]
- )*
- async fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read()
- && op.info().can_write()
- && op.info().can_copy() =>
$crate::copy::$test(op).await,
- Some(_) => {
- log::warn!("service {} doesn't support copy,
ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<copy_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read()
+ && op.info().can_write()
+ && op.info().can_copy() =>
RUNTIME.block_on($crate::copy::$test(op.clone())),
+ Some(_) => {
+ log::warn!("service {} doesn't support copy,
ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs
index 3c8d5347..648818dd 100644
--- a/core/tests/behavior/list.rs
+++ b/core/tests/behavior/list.rs
@@ -37,31 +37,28 @@ use super::utils::*;
macro_rules! behavior_list_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _list>] {
+ $(
+ #[test]
$(
- #[tokio::test]
- $(
- #[$meta]
- )*
- async fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read()
- && op.info().can_write()
- && (op.info().can_list()
- || op.info().can_scan()) =>
$crate::list::$test(op).await,
- Some(_) => {
- log::warn!("service {} doesn't support list,
ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<list_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read()
+ && op.info().can_write()
+ && (op.info().can_list()
+ || op.info().can_scan()) =>
RUNTIME.block_on($crate::list::$test(op.clone())),
+ Some(_) => {
+ log::warn!("service {} doesn't support list,
ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
@@ -98,13 +95,14 @@ pub async fn test_check(op: Operator) -> Result<()> {
/// List dir should return newly created file.
pub async fn test_list_dir(op: Operator) -> Result<()> {
- let path = uuid::Uuid::new_v4().to_string();
+ let parent = uuid::Uuid::new_v4().to_string();
+ let path = format!("{parent}/{}", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
let (content, size) = gen_bytes();
op.write(&path, content).await.expect("write must succeed");
- let mut obs = op.list("/").await?;
+ let mut obs = op.list(&format!("{parent}/")).await?;
let mut found = false;
while let Some(de) = obs.try_next().await? {
let meta = op.stat(de.path()).await?;
@@ -289,33 +287,37 @@ pub async fn test_scan_root(op: Operator) -> Result<()> {
.map(|v| v.path().to_string())
.collect::<HashSet<_>>();
- assert!(
- actual.is_empty(),
- "empty root should return empty, but got {:?}",
- actual
- );
+ assert!(!actual.contains("/"), "empty root should return itself");
+ assert!(!actual.contains(""), "empty root should return empty");
Ok(())
}
// Walk top down should output as expected
pub async fn test_scan(op: Operator) -> Result<()> {
+ let parent = uuid::Uuid::new_v4().to_string();
+
let expected = vec![
"x/", "x/y", "x/x/", "x/x/y", "x/x/x/", "x/x/x/y", "x/x/x/x/",
];
for path in expected.iter() {
if path.ends_with('/') {
- op.create_dir(path).await?;
+ op.create_dir(&format!("{parent}/{path}")).await?;
} else {
- op.write(path, "test_scan").await?;
+ op.write(&format!("{parent}/{path}"), "test_scan").await?;
}
}
- let w = op.scan("x/").await?;
+ let w = op.scan(&format!("{parent}/x/")).await?;
let actual = w
.try_collect::<Vec<_>>()
.await?
.into_iter()
- .map(|v| v.path().to_string())
+ .map(|v| {
+ v.path()
+ .strip_prefix(&format!("{parent}/"))
+ .unwrap()
+ .to_string()
+ })
.collect::<HashSet<_>>();
debug!("walk top down: {:?}", actual);
@@ -328,24 +330,29 @@ pub async fn test_scan(op: Operator) -> Result<()> {
// Remove all should remove all in this path.
pub async fn test_remove_all(op: Operator) -> Result<()> {
+ let parent = uuid::Uuid::new_v4().to_string();
+
let expected = vec![
"x/", "x/y", "x/x/", "x/x/y", "x/x/x/", "x/x/x/y", "x/x/x/x/",
];
for path in expected.iter() {
if path.ends_with('/') {
- op.create_dir(path).await?;
+ op.create_dir(&format!("{parent}/{path}")).await?;
} else {
- op.write(path, "test_remove_all").await?;
+ op.write(&format!("{parent}/{path}"), "test_scan").await?;
}
}
- op.remove_all("x/").await?;
+ op.remove_all(&format!("{parent}/x/")).await?;
for path in expected.iter() {
if path.ends_with('/') {
continue;
}
- assert!(!op.is_exist(path).await?, "{path} should be removed")
+ assert!(
+ !op.is_exist(&format!("{parent}/{path}")).await?,
+ "{parent}/{path} should be removed"
+ )
}
Ok(())
}
diff --git a/core/tests/behavior/list_only.rs b/core/tests/behavior/list_only.rs
index a8a292a4..a7e86a45 100644
--- a/core/tests/behavior/list_only.rs
+++ b/core/tests/behavior/list_only.rs
@@ -29,28 +29,25 @@ use opendal::Operator;
macro_rules! behavior_list_only_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _list_only>] {
+ $(
+ #[test]
$(
- #[tokio::test]
- $(
- #[$meta]
- )*
- async fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(false);
- match op {
- Some(op) if op.info().can_list() &&
!op.info().can_write() => $crate::list_only::$test(op).await,
- Some(_) => {
- log::warn!("service {} doesn't support list,
ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<list_only_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_list() &&
!op.info().can_write() =>
RUNTIME.block_on($crate::list_only::$test(op.clone())),
+ Some(_) => {
+ log::warn!("service {} doesn't support list,
ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs
index 63d63b16..c9038d05 100644
--- a/core/tests/behavior/main.rs
+++ b/core/tests/behavior/main.rs
@@ -46,32 +46,48 @@ mod utils;
/// Update function list while changed.
macro_rules! behavior_tests {
($($service:ident),*) => {
- $(
- // can_read && !can_write
- behavior_read_tests!($service);
- // can_read && !can_write && can_blocking
- behavior_blocking_read_tests!($service);
- // can_read && can_write
- behavior_write_tests!($service);
- // can_read && can_write && can_blocking
- behavior_blocking_write_tests!($service);
- // can_read && can_write && can_copy
- behavior_copy_tests!($service);
- // can read && can_write && can_blocking && can_copy
- behavior_blocking_copy_tests!($service);
- // can_read && can_write && can_move
- behavior_rename_tests!($service);
- // can_read && can_write && can_blocking && can_move
- behavior_blocking_rename_tests!($service);
- // can_read && can_write && can_list
- behavior_list_tests!($service);
- // can_read && can_write && can_presign
- behavior_presign_tests!($service);
- // can_read && can_write && can_blocking && can_list
- behavior_blocking_list_tests!($service);
- // can_list && !can_write
- behavior_list_only_tests!($service);
- )*
+ paste::item! {
+ $(
+ mod [<services_ $service:lower>] {
+ use once_cell::sync::Lazy;
+
+ static RUNTIME: Lazy<tokio::runtime::Runtime> =
Lazy::new(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ });
+ static OPERATOR: Lazy<Option<opendal::Operator>> =
Lazy::new(||
+
$crate::utils::init_service::<opendal::services::$service>()
+ );
+
+ // can_read && !can_write
+ behavior_read_tests!($service);
+ // can_read && !can_write && can_blocking
+ behavior_blocking_read_tests!($service);
+ // can_read && can_write
+ behavior_write_tests!($service);
+ // can_read && can_write && can_blocking
+ behavior_blocking_write_tests!($service);
+ // can_read && can_write && can_copy
+ behavior_copy_tests!($service);
+ // can read && can_write && can_blocking && can_copy
+ behavior_blocking_copy_tests!($service);
+ // can_read && can_write && can_move
+ behavior_rename_tests!($service);
+ // can_read && can_write && can_blocking && can_move
+ behavior_blocking_rename_tests!($service);
+ // can_read && can_write && can_list
+ behavior_list_tests!($service);
+ // can_read && can_write && can_presign
+ behavior_presign_tests!($service);
+ // can_read && can_write && can_blocking && can_list
+ behavior_blocking_list_tests!($service);
+ // can_list && !can_write
+ behavior_list_only_tests!($service);
+ }
+ )*
+ }
};
}
diff --git a/core/tests/behavior/presign.rs b/core/tests/behavior/presign.rs
index 8e42f902..ae4ae039 100644
--- a/core/tests/behavior/presign.rs
+++ b/core/tests/behavior/presign.rs
@@ -37,28 +37,25 @@ use super::utils::*;
macro_rules! behavior_presign_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _presign>] {
+ $(
+ #[test]
$(
- #[tokio::test]
- $(
- #[$meta]
- )*
- async fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read() &&
op.info().can_write() && op.info().can_presign() =>
$crate::presign::$test(op).await,
- Some(_) => {
- log::warn!("service {} doesn't support
presign, ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<presign_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read() &&
op.info().can_write() && op.info().can_presign() =>
RUNTIME.block_on($crate::presign::$test(op.clone())),
+ Some(_) => {
+ log::warn!("service {} doesn't support presign,
ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
diff --git a/core/tests/behavior/read_only.rs b/core/tests/behavior/read_only.rs
index 71054a74..6d3194cc 100644
--- a/core/tests/behavior/read_only.rs
+++ b/core/tests/behavior/read_only.rs
@@ -30,28 +30,25 @@ use sha2::Sha256;
macro_rules! behavior_read_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _read_only>] {
+ $(
+ #[test]
$(
- #[tokio::test]
- $(
- #[$meta]
- )*
- async fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(false);
- match op {
- Some(op) if op.info().can_read() &&
!op.info().can_write() => $crate::read_only::$test(op).await,
- Some(_) => {
- log::warn!("service {} doesn't support
read_only, ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<read_only_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read() &&
!op.info().can_write() =>
RUNTIME.block_on($crate::read_only::$test(op.clone())),
+ Some(_) => {
+ log::warn!("service {} doesn't support read_only,
ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
diff --git a/core/tests/behavior/rename.rs b/core/tests/behavior/rename.rs
index 14744869..cf4c0173 100644
--- a/core/tests/behavior/rename.rs
+++ b/core/tests/behavior/rename.rs
@@ -29,30 +29,27 @@ use super::utils::*;
macro_rules! behavior_rename_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _rename>] {
+ $(
+ #[test]
$(
- #[tokio::test]
- $(
- #[$meta]
- )*
- async fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read()
- && op.info().can_write()
- && op.info().can_rename() =>
$crate::rename::$test(op).await,
- Some(_) => {
- log::warn!("service {} doesn't support rename,
ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<rename_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read()
+ && op.info().can_write()
+ && op.info().can_rename() =>
RUNTIME.block_on($crate::rename::$test(op.clone())),
+ Some(_) => {
+ log::warn!("service {} doesn't support rename,
ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs
index b909d436..5592ecfe 100644
--- a/core/tests/behavior/utils.rs
+++ b/core/tests/behavior/utils.rs
@@ -34,7 +34,7 @@ use sha2::Sha256;
///
/// - If `opendal_{schema}_test` is on, construct a new Operator with given
root.
/// - Else, returns a `None` to represent no valid config for operator.
-pub fn init_service<B: Builder>(random_root: bool) -> Option<Operator> {
+pub fn init_service<B: Builder>() -> Option<Operator> {
let _ = env_logger::builder().is_test(true).try_init();
let _ = dotenvy::dotenv();
@@ -54,7 +54,9 @@ pub fn init_service<B: Builder>(random_root: bool) ->
Option<Operator> {
return None;
}
- if random_root {
+ // Use random root unless OPENDAL_DISABLE_RANDOM_ROOT is set to true.
+ let disable_random_root =
env::var("OPENDAL_DISABLE_RANDOM_ROOT").unwrap_or_default() == "true";
+ if !disable_random_root {
let root = format!(
"{}{}/",
cfg.get("root").cloned().unwrap_or_else(|| "/".to_string()),
diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs
index 7dffcb24..bf2d5c18 100644
--- a/core/tests/behavior/write.rs
+++ b/core/tests/behavior/write.rs
@@ -40,28 +40,25 @@ use super::utils::*;
macro_rules! behavior_write_test {
($service:ident, $($(#[$meta:meta])* $test:ident),*,) => {
paste::item! {
- mod [<services_ $service:lower _write>] {
+ $(
+ #[test]
$(
- #[tokio::test]
- $(
- #[$meta]
- )*
- async fn [< $test >]() -> anyhow::Result<()> {
- let op =
$crate::utils::init_service::<opendal::services::$service>(true);
- match op {
- Some(op) if op.info().can_read() &&
op.info().can_write() => $crate::write::$test(op).await,
- Some(_) => {
- log::warn!("service {} doesn't support write,
ignored", opendal::Scheme::$service);
- Ok(())
- },
- None => {
- log::warn!("service {} not initiated,
ignored", opendal::Scheme::$service);
- Ok(())
- }
+ #[$meta]
+ )*
+ fn [<write_ $test >]() -> anyhow::Result<()> {
+ match OPERATOR.as_ref() {
+ Some(op) if op.info().can_read() &&
op.info().can_write() => RUNTIME.block_on($crate::write::$test(op.clone())),
+ Some(_) => {
+ log::warn!("service {} doesn't support write,
ignored", opendal::Scheme::$service);
+ Ok(())
+ },
+ None => {
+ log::warn!("service {} not initiated, ignored",
opendal::Scheme::$service);
+ Ok(())
}
}
- )*
- }
+ }
+ )*
}
};
}
