This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch add-fuzz-writer-test in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 6ec01f7213d50cd650d57bf60eca26a5138ad4f1 Author: Xuanwo <[email protected]> AuthorDate: Mon Apr 24 18:02:51 2023 +0800 feat(tests): Add fuzz test for writer Signed-off-by: Xuanwo <[email protected]> --- core/src/services/fs/backend.rs | 1 + core/src/services/gcs/backend.rs | 1 + core/src/services/oss/backend.rs | 1 + core/src/services/s3/backend.rs | 2 + core/tests/behavior/utils.rs | 96 ++++++++++++++++++++++++++++++++++++++++ core/tests/behavior/write.rs | 28 ++++++++++++ 6 files changed, 129 insertions(+) diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index 4e78321c..5509663b 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -304,6 +304,7 @@ impl Accessor for FsBackend { read: true, read_can_seek: true, write: true, + write_without_content_length: true, create_dir: true, list: true, copy: true, diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index c2388dae..778c7781 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -377,6 +377,7 @@ impl Accessor for GcsBackend { read: true, read_can_next: true, write: true, + write_without_content_length: true, list: true, scan: true, copy: true, diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index 276fc32e..8748c7dd 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -367,6 +367,7 @@ impl Accessor for OssBackend { read: true, read_can_next: true, write: true, + write_without_content_length: true, list: true, scan: true, copy: true, diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index eed0335e..0adf125a 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -924,6 +924,8 @@ impl Accessor for S3Backend { read_with_override_content_disposition: true, write: true, + write_without_content_length: true, + list: true, scan: true, copy: true, diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs index 5592ecfe..fdf877d6 100644 --- a/core/tests/behavior/utils.rs +++ b/core/tests/behavior/utils.rs @@ -17,6 +17,9 @@ use std::collections::HashMap; use std::env; +use std::fmt; +use std::fmt::Debug; +use std::fmt::Formatter; use std::io::SeekFrom; use std::usize; @@ -279,3 +282,96 @@ impl ObjectReaderFuzzer { } } } + +/// ObjectWriterFuzzer is the fuzzer for object writer. +/// +/// We will generate random write operations to operate on object +/// write to check if the output is expected. +/// +/// # TODO +/// +/// This fuzzer only generate valid operations. +/// +/// In the future, we need to generate invalid operations to check if we +/// handled correctly. +pub struct ObjectWriterFuzzer { + name: String, + bs: Vec<u8>, + + size: Option<usize>, + cur: usize, + rng: ThreadRng, + actions: Vec<ObjectWriterAction>, +} + +#[derive(Clone)] +pub enum ObjectWriterAction { + Write(Bytes), +} + +impl Debug for ObjectWriterAction { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + ObjectWriterAction::Write(bs) => write!(f, "Write({})", bs.len()), + } + } +} + +impl ObjectWriterFuzzer { + /// Create a new fuzzer. + pub fn new(name: &str, size: Option<usize>) -> Self { + Self { + name: name.to_string(), + bs: Vec::new(), + + size, + cur: 0, + + rng: thread_rng(), + actions: vec![], + } + } + + /// Generate a new action. + pub fn fuzz(&mut self) -> ObjectWriterAction { + let max = if let Some(size) = self.size { + size - self.cur + } else { + // Set max to 1MiB + 1024 * 1024 + }; + + let size = self.rng.gen_range(0..max); + + let mut bs = vec![0; size]; + self.rng.fill_bytes(&mut bs); + + let bs = Bytes::from(bs); + self.bs.extend_from_slice(&bs); + self.cur += bs.len(); + + let action = ObjectWriterAction::Write(bs); + debug!("{} perform fuzz action: {:?}", self.name, action); + + self.actions.push(action.clone()); + + action + } + + /// Check if read operation is expected. + pub fn check(&mut self, actual_bs: &[u8]) { + assert_eq!( + self.bs.len(), + actual_bs.len(), + "check failed: expected len is different with actual len, actions: {:?}", + self.actions + ); + + assert_eq!( + format!("{:x}", Sha256::digest(&self.bs)), + format!("{:x}", Sha256::digest(actual_bs)), + "check failed: expected bs is different with actual bs, actions: {:?}", + self.actions, + ); + } +} diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs index 4b76128d..39a6d98d 100644 --- a/core/tests/behavior/write.rs +++ b/core/tests/behavior/write.rs @@ -104,6 +104,7 @@ macro_rules! behavior_write_tests { test_writer_write, test_writer_abort, test_writer_futures_copy, + test_fuzz_unsized_writer, ); )* }; @@ -885,3 +886,30 @@ pub async fn test_writer_futures_copy(op: Operator) -> Result<()> { op.delete(&path).await.expect("delete must succeed"); Ok(()) } + +/// Add test for unsized writer +pub async fn test_fuzz_unsized_writer(op: Operator) -> Result<()> { + if !op.info().capability().write_without_content_length { + warn!("{op:?} doesn't support write without content length, test skip"); + return Ok(()); + } + + let path = uuid::Uuid::new_v4().to_string(); + + let mut fuzzer = ObjectWriterFuzzer::new(&path, None); + + let mut w = op.writer(&path).await?; + + for _ in 0..100 { + match fuzzer.fuzz() { + ObjectWriterAction::Write(bs) => w.write(bs).await?, + } + } + w.close().await?; + + let content = op.read(&path).await?; + fuzzer.check(&content); + + op.delete(&path).await.expect("delete must succeed"); + Ok(()) +}
